ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Exempting the start_url from can_process_url

Browse Source
This commit is contained in:
Aravind Karnam
2024-11-23 14:59:14 +05:30
parent 3d52b551f2
commit 2226ef53c8
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawl4ai/scraper/bfs_scraper_strategy.py
View File

@@ -146,7 +146,7 @@ class BFSScraperStrategy(ScraperStrategy):
if self._cancel_event.is_set(): if self._cancel_event.is_set():
return None return None
if not await self.can_process_url(url): if depth!=0 and not await self.can_process_url(url):
self.stats.urls_skipped += 1 self.stats.urls_skipped += 1
return None return None