docs(multi-url): improve documentation clarity and update examples
- Restructure multi-URL crawling documentation with better formatting and examples - Update code examples to use new API syntax (arun_many) - Add detailed parameter explanations for RateLimiter and Dispatchers - Enhance CSS styling for better documentation readability - Fix outdated method calls in feature demo script BREAKING CHANGE: Updated dispatcher.run_urls() to crawler.arun_many() in examples
This commit is contained in:
@@ -85,10 +85,10 @@ async def demo_memory_dispatcher():
|
||||
)
|
||||
|
||||
print("\n🚀 Starting batch crawl...")
|
||||
results = await dispatcher.run_urls(
|
||||
results = await crawler.arun_many(
|
||||
urls=urls,
|
||||
crawler=crawler,
|
||||
config=crawler_config,
|
||||
dispatcher=dispatcher
|
||||
)
|
||||
print(f"\n✅ Completed {len(results)} URLs successfully")
|
||||
|
||||
@@ -115,15 +115,17 @@ async def demo_streaming_support():
|
||||
dispatcher = MemoryAdaptiveDispatcher(max_session_permit=3, check_interval=0.5)
|
||||
|
||||
print("Starting streaming crawl...")
|
||||
async for result in dispatcher.run_urls_stream(
|
||||
urls=urls, crawler=crawler, config=crawler_config
|
||||
async for result in await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=crawler_config,
|
||||
dispatcher=dispatcher
|
||||
):
|
||||
# Process each result as it arrives
|
||||
print(
|
||||
f"Received result for {result.url} - Success: {result.result.success}"
|
||||
f"Received result for {result.url} - Success: {result.success}"
|
||||
)
|
||||
if result.result.success:
|
||||
print(f"Content length: {len(result.result.markdown)}")
|
||||
if result.success:
|
||||
print(f"Content length: {len(result.markdown)}")
|
||||
|
||||
|
||||
async def demo_content_scraping():
|
||||
@@ -147,6 +149,8 @@ async def demo_content_scraping():
|
||||
print("Successfully scraped content using LXML strategy")
|
||||
|
||||
|
||||
|
||||
|
||||
async def demo_llm_markdown():
|
||||
"""
|
||||
4. LLM-Powered Markdown Generation Demo
|
||||
@@ -336,7 +340,7 @@ async def main():
|
||||
|
||||
# Efficiency & Speed Demos
|
||||
print("\n🚀 EFFICIENCY & SPEED DEMOS")
|
||||
await demo_memory_dispatcher()
|
||||
# await demo_memory_dispatcher()
|
||||
await demo_streaming_support()
|
||||
await demo_content_scraping()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user