ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor: Move adeep_crawl as method of crawler itself. Create attributes in CrawlResult to reconstruct the tree once deep crawling is completed

Browse Source
This commit is contained in:
Aravind Karnam
2025-01-29 15:58:21 +05:30
parent 84ffdaab9a
commit 9ef43bc5f0
2 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crawl4ai/async_webcrawler.py
View File

@@ -16,7 +16,6 @@ from .models import (
MarkdownGenerationResult, MarkdownGenerationResult,
CrawlerTaskResult, CrawlerTaskResult,
DispatchResult, DispatchResult,
DeepCrawlingProgress,
) )
from .async_database import async_db_manager from .async_database import async_db_manager
from .chunking_strategy import * # noqa: F403 from .chunking_strategy import * # noqa: F403

2
docs/scraper/deep_crawl_quickstart.py
View File

@@ -135,6 +135,7 @@ async def advanced_scraper_example():
try: try:
# Use streaming mode # Use streaming mode
results = []
result_generator = await crawler.adeep_crawl( result_generator = await crawler.adeep_crawl(
"https://techcrunch.com", "https://techcrunch.com",
strategy=bfs_strategy, strategy=bfs_strategy,
@@ -151,6 +152,7 @@ async def advanced_scraper_example():
logger.info( logger.info(
f"Processed at depth: {result.depth} with score: {result.score:.3f} : \n {result.url}" f"Processed at depth: {result.depth} with score: {result.score:.3f} : \n {result.url}"
) )
results.append(result)
else: else:
stats["errors"] += 1 stats["errors"] += 1
logger.error( logger.error(