feat(crawler): add experimental parameters support and optimize browser handling

Add experimental parameters dictionary to CrawlerRunConfig to support beta features
Make CSP nonce headers optional via experimental config
Remove default cookie injection
Clean up browser context creation code
Improve code formatting in API handler

BREAKING CHANGE: Default cookie injection has been removed from page initialization
This commit is contained in:
UncleCode
2025-03-14 14:39:24 +08:00
parent 6e3c048328
commit 7884a98be7
4 changed files with 32 additions and 24 deletions

View File

@@ -391,7 +391,10 @@ async def handle_crawl_request(
async with AsyncWebCrawler(config=browser_config) as crawler:
results = []
func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
partial_func = partial(func, urls[0] if len(urls) == 1 else urls, config=crawler_config, dispatcher=dispatcher)
partial_func = partial(func,
urls[0] if len(urls) == 1 else urls,
config=crawler_config,
dispatcher=dispatcher)
results = await partial_func()
return {
"success": True,