ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#1564 fix: Improve error handling in browser configuration serialization and cleanup logic

Browse Source
This commit is contained in:
AHMET YILMAZ
2025-10-27 17:02:26 +08:00
parent 00e9904609
commit a3b02be5c3
1 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
deploy/docker/crawler_pool.py
View File

@@ -38,7 +38,11 @@ IDLE_TTL = (
def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str: def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":")) try:
config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":"))
except (TypeError, ValueError):
# Fallback to string representation if JSON serialization fails
config_payload = str(cfg.to_dict())
adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter" adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter"
payload = f"{config_payload}:{adapter_name}" payload = f"{config_payload}:{adapter_name}"
return hashlib.sha1(payload.encode()).hexdigest() return hashlib.sha1(payload.encode()).hexdigest()
@@ -47,6 +51,7 @@ def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
async def get_crawler( async def get_crawler(
cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None
) -> AsyncWebCrawler: ) -> AsyncWebCrawler:
sig = None
try: try:
sig = _sig(cfg, adapter) sig = _sig(cfg, adapter)
async with LOCK: async with LOCK:
@@ -82,12 +87,13 @@ async def get_crawler(
except Exception as e: except Exception as e:
raise RuntimeError(f"Failed to start browser: {e}") raise RuntimeError(f"Failed to start browser: {e}")
finally: finally:
if sig in POOL: if sig:
LAST_USED[sig] = time.time() if sig in POOL:
else: LAST_USED[sig] = time.time()
# If we failed to start the browser, we should remove it from the pool else:
POOL.pop(sig, None) # If we failed to start the browser, we should remove it from the pool
LAST_USED.pop(sig, None) POOL.pop(sig, None)
LAST_USED.pop(sig, None)
# If we failed to start the browser, we should remove it from the pool # If we failed to start the browser, we should remove it from the pool