Refactor: Move adeep_crawl as method of crawler itself. Create attributes in CrawlResult to reconstruct the tree once deep crawling is completed

crawl4ai/async_webcrawler.py

@@ -16,7 +16,6 @@ from .models import (
     MarkdownGenerationResult,
     CrawlerTaskResult,
     DispatchResult,
-    DeepCrawlingProgress,
 )
 from .async_database import async_db_manager
 from .chunking_strategy import *  # noqa: F403

docs/scraper/deep_crawl_quickstart.py

@@ -135,6 +135,7 @@ async def advanced_scraper_example():
 
         try:
             # Use streaming mode
+            results = []
             result_generator = await crawler.adeep_crawl(
                 "https://techcrunch.com",
                 strategy=bfs_strategy,
@@ -151,6 +152,7 @@ async def advanced_scraper_example():
                     logger.info(
                         f"Processed at depth: {result.depth} with score: {result.score:.3f} : \n {result.url}"
                     )
+                    results.append(result)
                 else:
                     stats["errors"] += 1
                     logger.error(