ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add context caching to create_isolated_context branch

Browse Source 
Uses contexts_by_config cache (same as non-CDP mode) to reuse contexts
for multiple URLs with same config. Still creates new page per crawl
for navigation isolation. Benefits batch/deep crawls.
This commit is contained in:
unclecode
2025-12-13 08:58:21 +00:00
parent 55eb968a8d
commit ecedb6113e
1 changed files with 15 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
crawl4ai/browser_manager.py
View File

@@ -1149,12 +1149,22 @@ class BrowserManager:
# If using a managed browser, just grab the shared default_context # If using a managed browser, just grab the shared default_context
if self.config.use_managed_browser: if self.config.use_managed_browser:
# If create_isolated_context is True, create a fresh context for this crawl # If create_isolated_context is True, create isolated contexts for concurrent crawls
# This is essential for concurrent crawls on the same browser to prevent # Uses the same caching mechanism as non-CDP mode: cache context by config signature,
# navigation conflicts (multiple crawls sharing the same page) # but always create a new page. This prevents navigation conflicts while allowing
# context reuse for multiple URLs with the same config (e.g., batch/deep crawls).
if self.config.create_isolated_context: if self.config.create_isolated_context:
config_signature = self._make_config_signature(crawlerRunConfig)
async with self._contexts_lock:
if config_signature in self.contexts_by_config:
context = self.contexts_by_config[config_signature]
else:
context = await self.create_browser_context(crawlerRunConfig) context = await self.create_browser_context(crawlerRunConfig)
await self.setup_context(context, crawlerRunConfig) await self.setup_context(context, crawlerRunConfig)
self.contexts_by_config[config_signature] = context
# Always create a new page for each crawl (isolation for navigation)
page = await context.new_page() page = await context.new_page()
await self._apply_stealth_to_page(page) await self._apply_stealth_to_page(page)
elif self.config.storage_state: elif self.config.storage_state: