ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix "Future attached to a different loop" error by ensuring tasks are created in the correct event loop

Browse Source 
- Explicitly retrieve and use the correct event loop when creating tasks to avoid cross-loop issues.
- Ensures proper task scheduling in environments with multiple event loops.
This commit is contained in:
Aravind Karnam
2024-11-26 14:05:02 +05:30
parent a98d51a62c
commit a888c91790
3 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crawl4ai/async_crawler_strategy.py
View File

@@ -68,7 +68,8 @@ class ManagedBrowser:
stderr=subprocess.PIPE stderr=subprocess.PIPE
) )
# Monitor browser process output for errors # Monitor browser process output for errors
asyncio.create_task(self._monitor_browser_process()) loop = asyncio.get_event_loop()
loop.create_task(self._monitor_browser_process())
await asyncio.sleep(2) # Give browser time to start await asyncio.sleep(2) # Give browser time to start
return f"http://localhost:{self.debugging_port}" return f"http://localhost:{self.debugging_port}"
except Exception as e: except Exception as e:

3
crawl4ai/scraper/bfs_scraper_strategy.py
View File

@@ -264,7 +264,8 @@ class BFSScraperStrategy(ScraperStrategy):
self.stats.current_depth = depth self.stats.current_depth = depth
if parallel_processing: if parallel_processing:
task = asyncio.create_task( loop = asyncio.get_event_loop()
task = loop.create_task(
self.process_url(url, depth, crawler, queue, visited, depths) self.process_url(url, depth, crawler, queue, visited, depths)
) )
pending_tasks.add(task) pending_tasks.add(task)

6
main.py
View File

@@ -125,7 +125,8 @@ class TaskManager:
self.cleanup_task = None self.cleanup_task = None
async def start(self): async def start(self):
self.cleanup_task = asyncio.create_task(self._cleanup_loop()) loop = asyncio.get_event_loop()
self.cleanup_task = loop.create_task(self._cleanup_loop())
async def stop(self): async def stop(self):
if self.cleanup_task: if self.cleanup_task:
@@ -231,7 +232,8 @@ class CrawlerService:
async def start(self): async def start(self):
await self.task_manager.start() await self.task_manager.start()
self._processing_task = asyncio.create_task(self._process_queue()) loop = asyncio.get_event_loop()
self._processing_task = loop.create_task(self._process_queue())
async def stop(self): async def stop(self):
if self._processing_task: if self._processing_task: