From a98d51a62cd9f1ee27bf9459c88f73b1b7956d37 Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Tue, 26 Nov 2024 11:11:49 +0530 Subject: [PATCH] Remove the can_process_url check from _process_links since it's already being checked in process_url --- crawl4ai/scraper/bfs_scraper_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawl4ai/scraper/bfs_scraper_strategy.py b/crawl4ai/scraper/bfs_scraper_strategy.py index 3109e96d..3a6d09a5 100644 --- a/crawl4ai/scraper/bfs_scraper_strategy.py +++ b/crawl4ai/scraper/bfs_scraper_strategy.py @@ -198,7 +198,7 @@ class BFSScraperStrategy(ScraperStrategy): links_to_process += result.links["external"] for link in links_to_process: url = link['href'] - if url not in visited and await self.can_process_url(url,depth): + if url not in visited: new_depth = depths[source_url] + 1 if new_depth <= self.max_depth: if self.url_scorer: