From f34b4878cf1e5106a9ac1067754fd427f6bd665d Mon Sep 17 00:00:00 2001 From: Aravind Karnam Date: Tue, 28 Jan 2025 10:00:01 +0530 Subject: [PATCH] fix: code formatting --- crawl4ai/scraper/bfs_scraper_strategy.py | 4 +++- docs/scraper/scraper_quickstart.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crawl4ai/scraper/bfs_scraper_strategy.py b/crawl4ai/scraper/bfs_scraper_strategy.py index 962c6bee..11568ed7 100644 --- a/crawl4ai/scraper/bfs_scraper_strategy.py +++ b/crawl4ai/scraper/bfs_scraper_strategy.py @@ -154,7 +154,9 @@ class BFSScraperStrategy(ScraperStrategy): # Fill batch with available jobs while len(jobs) < SCRAPER_BATCH_SIZE and not queue.empty(): score, depth, url = await queue.get() - if url not in active_crawls: # Only add if not currently processing + if ( + url not in active_crawls + ): # Only add if not currently processing jobs.append((score, depth, url)) active_crawls.add(url) self.stats.current_depth = depth diff --git a/docs/scraper/scraper_quickstart.py b/docs/scraper/scraper_quickstart.py index da4aef7f..893dd5ce 100644 --- a/docs/scraper/scraper_quickstart.py +++ b/docs/scraper/scraper_quickstart.py @@ -129,9 +129,11 @@ async def advanced_scraper_example(): ) # Create crawler and scraper - async with AsyncWebScraper(crawler_config=CrawlerRunConfig(bypass_cache=True), - browser_config=browser_config, - strategy=bfs_strategy) as scraper: + async with AsyncWebScraper( + crawler_config=CrawlerRunConfig(bypass_cache=True), + browser_config=browser_config, + strategy=bfs_strategy, + ) as scraper: # Track statistics stats = {"processed": 0, "errors": 0, "total_size": 0}