docs(multi-url): improve documentation clarity and update examples

- Restructure multi-URL crawling documentation with better formatting and examples
- Update code examples to use new API syntax (arun_many)
- Add detailed parameter explanations for RateLimiter and Dispatchers
- Enhance CSS styling for better documentation readability
- Fix outdated method calls in feature demo script

BREAKING CHANGE: Updated dispatcher.run_urls() to crawler.arun_many() in examples
This commit is contained in:
UncleCode
2025-01-23 22:33:36 +08:00
parent 6dc01eae3a
commit 6a01008a2b
3 changed files with 201 additions and 28 deletions

View File

@@ -85,10 +85,10 @@ async def demo_memory_dispatcher():
)
print("\n🚀 Starting batch crawl...")
results = await dispatcher.run_urls(
results = await crawler.arun_many(
urls=urls,
crawler=crawler,
config=crawler_config,
dispatcher=dispatcher
)
print(f"\n✅ Completed {len(results)} URLs successfully")
@@ -115,15 +115,17 @@ async def demo_streaming_support():
dispatcher = MemoryAdaptiveDispatcher(max_session_permit=3, check_interval=0.5)
print("Starting streaming crawl...")
async for result in dispatcher.run_urls_stream(
urls=urls, crawler=crawler, config=crawler_config
async for result in await crawler.arun_many(
urls=urls,
config=crawler_config,
dispatcher=dispatcher
):
# Process each result as it arrives
print(
f"Received result for {result.url} - Success: {result.result.success}"
f"Received result for {result.url} - Success: {result.success}"
)
if result.result.success:
print(f"Content length: {len(result.result.markdown)}")
if result.success:
print(f"Content length: {len(result.markdown)}")
async def demo_content_scraping():
@@ -147,6 +149,8 @@ async def demo_content_scraping():
print("Successfully scraped content using LXML strategy")
async def demo_llm_markdown():
"""
4. LLM-Powered Markdown Generation Demo
@@ -336,7 +340,7 @@ async def main():
# Efficiency & Speed Demos
print("\n🚀 EFFICIENCY & SPEED DEMOS")
await demo_memory_dispatcher()
# await demo_memory_dispatcher()
await demo_streaming_support()
await demo_content_scraping()