ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove the can_process_url check from _process_links since it's already being checked in process_url

Browse Source
This commit is contained in:
Aravind Karnam
2024-11-26 11:11:49 +05:30
parent ee3001b1f7
commit a98d51a62c
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawl4ai/scraper/bfs_scraper_strategy.py
View File

@@ -198,7 +198,7 @@ class BFSScraperStrategy(ScraperStrategy):
links_to_process += result.links["external"] links_to_process += result.links["external"]
for link in links_to_process: for link in links_to_process:
url = link['href'] url = link['href']
if url not in visited and await self.can_process_url(url,depth): if url not in visited:
new_depth = depths[source_url] + 1 new_depth = depths[source_url] + 1
if new_depth <= self.max_depth: if new_depth <= self.max_depth:
if self.url_scorer: if self.url_scorer: