fix: code formatting

This commit is contained in:
Aravind Karnam
2025-01-28 10:00:01 +05:30
parent d9324e3454
commit f34b4878cf
2 changed files with 8 additions and 4 deletions

View File

@@ -154,7 +154,9 @@ class BFSScraperStrategy(ScraperStrategy):
# Fill batch with available jobs
while len(jobs) < SCRAPER_BATCH_SIZE and not queue.empty():
score, depth, url = await queue.get()
if url not in active_crawls: # Only add if not currently processing
if (
url not in active_crawls
): # Only add if not currently processing
jobs.append((score, depth, url))
active_crawls.add(url)
self.stats.current_depth = depth

View File

@@ -129,9 +129,11 @@ async def advanced_scraper_example():
)
# Create crawler and scraper
async with AsyncWebScraper(crawler_config=CrawlerRunConfig(bypass_cache=True),
browser_config=browser_config,
strategy=bfs_strategy) as scraper:
async with AsyncWebScraper(
crawler_config=CrawlerRunConfig(bypass_cache=True),
browser_config=browser_config,
strategy=bfs_strategy,
) as scraper:
# Track statistics
stats = {"processed": 0, "errors": 0, "total_size": 0}