ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs(examples): update proxy rotation demo and disable other demos

Browse Source 
Modify proxy rotation example to include empty user agent setting and comment out other demo functions for focused testing. This change simplifies the demo file to focus specifically on proxy rotation functionality.

No breaking changes.
This commit is contained in:
UncleCode
2025-01-25 21:52:35 +08:00
parent 4d7f91b378
commit 97796f39d2
1 changed files with 8 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
docs/examples/v0_4_3b2_features_demo.py
View File

@@ -299,7 +299,6 @@ async def demo_proxy_rotation():
print(f"Error loading proxy: {e}") print(f"Error loading proxy: {e}")
return None return None
# Create 10 test requests to httpbin # Create 10 test requests to httpbin
urls = ["https://httpbin.org/ip"] * 2 urls = ["https://httpbin.org/ip"] * 2
@@ -314,7 +313,7 @@ async def demo_proxy_rotation():
continue continue
# Create new config with proxy # Create new config with proxy
current_config = run_config.clone(proxy_config=proxy) current_config = run_config.clone(proxy_config=proxy, user_agent="")
result = await crawler.arun(url=url, config=current_config) result = await crawler.arun(url=url, config=current_config)
if result.success: if result.success:
@@ -334,18 +333,18 @@ async def main():
# Efficiency & Speed Demos # Efficiency & Speed Demos
print("\n🚀 EFFICIENCY & SPEED DEMOS") print("\n🚀 EFFICIENCY & SPEED DEMOS")
await demo_memory_dispatcher() # await demo_memory_dispatcher()
await demo_streaming_support() # await demo_streaming_support()
await demo_content_scraping() # await demo_content_scraping()
# # LLM Integration Demos # # LLM Integration Demos
print("\n🤖 LLM INTEGRATION DEMOS") print("\n🤖 LLM INTEGRATION DEMOS")
await demo_json_schema_generation() # await demo_json_schema_generation()
await demo_llm_markdown() # await demo_llm_markdown()
# # Core Improvements # # Core Improvements
print("\n🔧 CORE IMPROVEMENT DEMOS") print("\n🔧 CORE IMPROVEMENT DEMOS")
await demo_robots_compliance() # await demo_robots_compliance()
await demo_proxy_rotation() await demo_proxy_rotation()
if __name__ == "__main__": if __name__ == "__main__":