Add demo script for proxy rotation and quick test suite

- Implemented demo_proxy_rotation.py to showcase various proxy rotation strategies and their integration with the API.
- Included multiple demos demonstrating round robin, random, least used, failure-aware, and streaming strategies.
- Added error handling and real-world scenario examples for e-commerce price monitoring.
- Created quick_proxy_test.py to validate API integration without real proxies, testing parameter acceptance, invalid strategy rejection, and optional parameters.
- Ensured both scripts provide informative output and usage instructions.
This commit is contained in:
AHMET YILMAZ
2025-10-06 13:40:38 +08:00
parent 5dc34dd210
commit f00e8cbf35
7 changed files with 1706 additions and 5 deletions

View File

@@ -607,6 +607,10 @@ async def crawl(
hooks_config=hooks_config,
anti_bot_strategy=crawl_request.anti_bot_strategy,
headless=crawl_request.headless,
proxy_rotation_strategy=crawl_request.proxy_rotation_strategy,
proxies=crawl_request.proxies,
proxy_failure_threshold=crawl_request.proxy_failure_threshold,
proxy_recovery_time=crawl_request.proxy_recovery_time,
)
# check if all of the results are not successful
if all(not result["success"] for result in results["results"]):
@@ -646,6 +650,10 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):
hooks_config=hooks_config,
anti_bot_strategy=crawl_request.anti_bot_strategy,
headless=crawl_request.headless,
proxy_rotation_strategy=crawl_request.proxy_rotation_strategy,
proxies=crawl_request.proxies,
proxy_failure_threshold=crawl_request.proxy_failure_threshold,
proxy_recovery_time=crawl_request.proxy_recovery_time,
)
# Add hooks info to response headers if available