From a888c91790e8234179f8253f6bbb66dcf401bbca Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Tue, 26 Nov 2024 14:05:02 +0530 Subject: [PATCH] Fix "Future attached to a different loop" error by ensuring tasks are created in the correct event loop - Explicitly retrieve and use the correct event loop when creating tasks to avoid cross-loop issues. - Ensures proper task scheduling in environments with multiple event loops. --- crawl4ai/async_crawler_strategy.py | 3 ++- crawl4ai/scraper/bfs_scraper_strategy.py | 3 ++- main.py | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 3f332eb0..df23c43e 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -68,7 +68,8 @@ class ManagedBrowser: stderr=subprocess.PIPE ) # Monitor browser process output for errors - asyncio.create_task(self._monitor_browser_process()) + loop = asyncio.get_event_loop() + loop.create_task(self._monitor_browser_process()) await asyncio.sleep(2) # Give browser time to start return f"http://localhost:{self.debugging_port}" except Exception as e: diff --git a/crawl4ai/scraper/bfs_scraper_strategy.py b/crawl4ai/scraper/bfs_scraper_strategy.py index 3a6d09a5..73a4f8ae 100644 --- a/crawl4ai/scraper/bfs_scraper_strategy.py +++ b/crawl4ai/scraper/bfs_scraper_strategy.py @@ -264,7 +264,8 @@ class BFSScraperStrategy(ScraperStrategy): self.stats.current_depth = depth if parallel_processing: - task = asyncio.create_task( + loop = asyncio.get_event_loop() + task = loop.create_task( self.process_url(url, depth, crawler, queue, visited, depths) ) pending_tasks.add(task) diff --git a/main.py b/main.py index 6d217410..bc5dfe7e 100644 --- a/main.py +++ b/main.py @@ -125,7 +125,8 @@ class TaskManager: self.cleanup_task = None async def start(self): - self.cleanup_task = asyncio.create_task(self._cleanup_loop()) + loop = asyncio.get_event_loop() + self.cleanup_task = loop.create_task(self._cleanup_loop()) async def stop(self): if self.cleanup_task: @@ -231,7 +232,8 @@ class CrawlerService: async def start(self): await self.task_manager.start() - self._processing_task = asyncio.create_task(self._process_queue()) + loop = asyncio.get_event_loop() + self._processing_task = loop.create_task(self._process_queue()) async def stop(self): if self._processing_task: