ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Move the creation of crawler outside the main loop

Browse Source
This commit is contained in:
Aravind Karnam
2025-01-27 18:31:13 +05:30
parent 0ff95c83bc
commit d9324e3454
1 changed files with 35 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crawl4ai/scraper/bfs_scraper_strategy.py
View File

@@ -135,6 +135,10 @@ class BFSScraperStrategy(ScraperStrategy):
visited: Set[str] = set() visited: Set[str] = set()
depths = {start_url: 0} depths = {start_url: 0}
active_crawls = set() # Track URLs currently being processed active_crawls = set() # Track URLs currently being processed
async with AsyncWebCrawler(
config=browser_config,
verbose=True,
) as crawler:
try: try:
while ( while (
not queue.empty() or active_crawls not queue.empty() or active_crawls
@@ -161,10 +165,6 @@ class BFSScraperStrategy(ScraperStrategy):
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
continue continue
# Process batch # Process batch
async with AsyncWebCrawler(
config=browser_config,
verbose=True,
) as crawler:
try: try:
async for result in await crawler.arun_many( async for result in await crawler.arun_many(
urls=[url for _, _, url in jobs], urls=[url for _, _, url in jobs],