From 97796f39d27f3a0f9ee62512f72dafb2e630a29e Mon Sep 17 00:00:00 2001 From: UncleCode Date: Sat, 25 Jan 2025 21:52:35 +0800 Subject: [PATCH] docs(examples): update proxy rotation demo and disable other demos Modify proxy rotation example to include empty user agent setting and comment out other demo functions for focused testing. This change simplifies the demo file to focus specifically on proxy rotation functionality. No breaking changes. --- docs/examples/v0_4_3b2_features_demo.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/examples/v0_4_3b2_features_demo.py b/docs/examples/v0_4_3b2_features_demo.py index 7771c3f8..3b604c62 100644 --- a/docs/examples/v0_4_3b2_features_demo.py +++ b/docs/examples/v0_4_3b2_features_demo.py @@ -297,8 +297,7 @@ async def demo_proxy_rotation(): } except Exception as e: print(f"Error loading proxy: {e}") - return None - + return None # Create 10 test requests to httpbin urls = ["https://httpbin.org/ip"] * 2 @@ -314,7 +313,7 @@ async def demo_proxy_rotation(): continue # Create new config with proxy - current_config = run_config.clone(proxy_config=proxy) + current_config = run_config.clone(proxy_config=proxy, user_agent="") result = await crawler.arun(url=url, config=current_config) if result.success: @@ -334,18 +333,18 @@ async def main(): # Efficiency & Speed Demos print("\nšŸš€ EFFICIENCY & SPEED DEMOS") - await demo_memory_dispatcher() - await demo_streaming_support() - await demo_content_scraping() + # await demo_memory_dispatcher() + # await demo_streaming_support() + # await demo_content_scraping() # # LLM Integration Demos print("\nšŸ¤– LLM INTEGRATION DEMOS") - await demo_json_schema_generation() - await demo_llm_markdown() + # await demo_json_schema_generation() + # await demo_llm_markdown() # # Core Improvements print("\nšŸ”§ CORE IMPROVEMENT DEMOS") - await demo_robots_compliance() + # await demo_robots_compliance() await demo_proxy_rotation() if __name__ == "__main__":