ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#1564 fix: Improve error handling in browser configuration serialization and cleanup logic

Browse Source
This commit is contained in:
AHMET YILMAZ
2025-10-27 17:02:26 +08:00
parent 00e9904609
commit a3b02be5c3
1 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
deploy/docker/crawler_pool.py
View File

@@ -38,7 +38,11 @@ IDLE_TTL = (
def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str: def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
try:
config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":")) config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":"))
except (TypeError, ValueError):
# Fallback to string representation if JSON serialization fails
config_payload = str(cfg.to_dict())
adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter" adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter"
payload = f"{config_payload}:{adapter_name}" payload = f"{config_payload}:{adapter_name}"
return hashlib.sha1(payload.encode()).hexdigest() return hashlib.sha1(payload.encode()).hexdigest()
@@ -47,6 +51,7 @@ def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
async def get_crawler( async def get_crawler(
cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None
) -> AsyncWebCrawler: ) -> AsyncWebCrawler:
sig = None
try: try:
sig = _sig(cfg, adapter) sig = _sig(cfg, adapter)
async with LOCK: async with LOCK:
@@ -82,6 +87,7 @@ async def get_crawler(
except Exception as e: except Exception as e:
raise RuntimeError(f"Failed to start browser: {e}") raise RuntimeError(f"Failed to start browser: {e}")
finally: finally:
if sig:
if sig in POOL: if sig in POOL:
LAST_USED[sig] = time.time() LAST_USED[sig] = time.time()
else: else: