diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py index 5e250ca6..26e8a9a1 100644 --- a/crawl4ai/deep_crawling/bff_strategy.py +++ b/crawl4ai/deep_crawling/bff_strategy.py @@ -177,7 +177,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy): base_url = normalize_url_for_deep_crawl(url, source_url) if base_url in visited: continue - if not await self.can_process_url(url, new_depth): + if not await self.can_process_url(base_url, new_depth): self.stats.urls_skipped += 1 continue diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index dab94532..dfb75927 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -170,7 +170,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy): base_url = normalize_url_for_deep_crawl(url, source_url) if base_url in visited: continue - if not await self.can_process_url(url, next_depth): + if not await self.can_process_url(base_url, next_depth): self.stats.urls_skipped += 1 continue diff --git a/crawl4ai/deep_crawling/dfs_strategy.py b/crawl4ai/deep_crawling/dfs_strategy.py index 5e592fc1..3e4987f2 100644 --- a/crawl4ai/deep_crawling/dfs_strategy.py +++ b/crawl4ai/deep_crawling/dfs_strategy.py @@ -300,7 +300,7 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy): if not normalized_url or normalized_url in seen: continue - if not await self.can_process_url(raw_url, next_depth): + if not await self.can_process_url(normalized_url, next_depth): self.stats.urls_skipped += 1 continue