From 2226ef53c84fe116a5cc6c15f1cd0fab178715f9 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Sat, 23 Nov 2024 14:59:14 +0530 Subject: [PATCH] fix: Exempting the start_url from can_process_url --- crawl4ai/scraper/bfs_scraper_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawl4ai/scraper/bfs_scraper_strategy.py b/crawl4ai/scraper/bfs_scraper_strategy.py index 72a86203..b87412ef 100644 --- a/crawl4ai/scraper/bfs_scraper_strategy.py +++ b/crawl4ai/scraper/bfs_scraper_strategy.py @@ -146,7 +146,7 @@ class BFSScraperStrategy(ScraperStrategy): if self._cancel_event.is_set(): return None - if not await self.can_process_url(url): + if depth!=0 and not await self.can_process_url(url): self.stats.urls_skipped += 1 return None