Add demo script for proxy rotation and quick test suite
- Implemented demo_proxy_rotation.py to showcase various proxy rotation strategies and their integration with the API. - Included multiple demos demonstrating round robin, random, least used, failure-aware, and streaming strategies. - Added error handling and real-world scenario examples for e-commerce price monitoring. - Created quick_proxy_test.py to validate API integration without real proxies, testing parameter acceptance, invalid strategy rejection, and optional parameters. - Ensured both scripts provide informative output and usage instructions.
This commit is contained in:
@@ -607,6 +607,10 @@ async def crawl(
|
||||
hooks_config=hooks_config,
|
||||
anti_bot_strategy=crawl_request.anti_bot_strategy,
|
||||
headless=crawl_request.headless,
|
||||
proxy_rotation_strategy=crawl_request.proxy_rotation_strategy,
|
||||
proxies=crawl_request.proxies,
|
||||
proxy_failure_threshold=crawl_request.proxy_failure_threshold,
|
||||
proxy_recovery_time=crawl_request.proxy_recovery_time,
|
||||
)
|
||||
# check if all of the results are not successful
|
||||
if all(not result["success"] for result in results["results"]):
|
||||
@@ -646,6 +650,10 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):
|
||||
hooks_config=hooks_config,
|
||||
anti_bot_strategy=crawl_request.anti_bot_strategy,
|
||||
headless=crawl_request.headless,
|
||||
proxy_rotation_strategy=crawl_request.proxy_rotation_strategy,
|
||||
proxies=crawl_request.proxies,
|
||||
proxy_failure_threshold=crawl_request.proxy_failure_threshold,
|
||||
proxy_recovery_time=crawl_request.proxy_recovery_time,
|
||||
)
|
||||
|
||||
# Add hooks info to response headers if available
|
||||
|
||||
Reference in New Issue
Block a user