ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(crawler): add experimental parameters support and optimize browser handling

Browse Source 
Add experimental parameters dictionary to CrawlerRunConfig to support beta features
Make CSP nonce headers optional via experimental config
Remove default cookie injection
Clean up browser context creation code
Improve code formatting in API handler

BREAKING CHANGE: Default cookie injection has been removed from page initialization
This commit is contained in:
UncleCode
2025-03-14 14:39:24 +08:00
parent 6e3c048328
commit 7884a98be7
4 changed files with 32 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
crawl4ai/async_crawler_strategy.py
View File

@@ -507,10 +507,12 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
# Get page for session
page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
# await page.goto(URL)
# Add default cookie
await context.add_cookies(
[{"name": "cookiesEnabled", "value": "true", "url": url}]
)
# await context.add_cookies(
# [{"name": "cookiesEnabled", "value": "true", "url": url}]
# )
# Handle navigator overrides
if config.override_navigator or config.simulate_user or config.magic:
@@ -562,14 +564,15 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
try:
# Generate a unique nonce for this request
nonce = hashlib.sha256(os.urandom(32)).hexdigest()
if config.experimental.get("use_csp_nonce", False):
nonce = hashlib.sha256(os.urandom(32)).hexdigest()
# Add CSP headers to the request
await page.set_extra_http_headers(
{
"Content-Security-Policy": f"default-src 'self'; script-src 'self' 'nonce-{nonce}' 'strict-dynamic'"
}
)
# Add CSP headers to the request
await page.set_extra_http_headers(
{
"Content-Security-Policy": f"default-src 'self'; script-src 'self' 'nonce-{nonce}' 'strict-dynamic'"
}
)
response = await page.goto(
url, wait_until=config.wait_until, timeout=config.page_timeout