fix: Exempting the start_url from can_process_url

2024-11-23 14:59:14 +05:30
parent 3d52b551f2
commit 2226ef53c8
1 changed files with 1 additions and 1 deletions
--- a/crawl4ai/scraper/bfs_scraper_strategy.py
+++ b/crawl4ai/scraper/bfs_scraper_strategy.py
@@ -146,7 +146,7 @@ class BFSScraperStrategy(ScraperStrategy):
        if self._cancel_event.is_set():
            return None
            
-        if not await self.can_process_url(url):
+        if depth!=0 and not await self.can_process_url(url):
            self.stats.urls_skipped += 1
            return None