#!/usr/bin/env python3
"""
Prefetch Mode and Two-Phase Crawling Example
Prefetch mode is a fast path that skips heavy processing and returns
only HTML + links. This is ideal for:
- Site mapping: Quickly discover all URLs
- Selective crawling: Find URLs first, then process only what you need
- Link validation: Check which pages exist without full processing
- Crawl planning: Estimate size before committing resources
Key concept:
- `prefetch=True` in CrawlerRunConfig enables fast link-only extraction
- Skips: markdown generation, content scraping, media extraction, LLM extraction
- Returns: HTML and links dictionary
Performance benefit: ~5-10x faster than full processing
"""
import asyncio
import time
from typing import List, Dict
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
async def example_basic_prefetch():
"""
Example 1: Basic prefetch mode.
Shows how prefetch returns HTML and links without heavy processing.
"""
print("\n" + "=" * 60)
print("Example 1: Basic Prefetch Mode")
print("=" * 60)
async with AsyncWebCrawler(verbose=False) as crawler:
# Enable prefetch mode
config = CrawlerRunConfig(prefetch=True)
print("\nFetching with prefetch=True...")
result = await crawler.arun("https://books.toscrape.com", config=config)
print(f"\nResult summary:")
print(f" Success: {result.success}")
print(f" HTML length: {len(result.html) if result.html else 0} chars")
print(f" Internal links: {len(result.links.get('internal', []))}")
print(f" External links: {len(result.links.get('external', []))}")
# These should be None/empty in prefetch mode
print(f"\n Skipped processing:")
print(f" Markdown: {result.markdown}")
print(f" Cleaned HTML: {result.cleaned_html}")
print(f" Extracted content: {result.extracted_content}")
# Show some discovered links
internal_links = result.links.get("internal", [])
if internal_links:
print(f"\n Sample internal links:")
for link in internal_links[:5]:
print(f" - {link['href'][:60]}...")
async def example_performance_comparison():
"""
Example 2: Compare prefetch vs full processing performance.
"""
print("\n" + "=" * 60)
print("Example 2: Performance Comparison")
print("=" * 60)
url = "https://books.toscrape.com"
async with AsyncWebCrawler(verbose=False) as crawler:
# Warm up - first request is slower due to browser startup
await crawler.arun(url, config=CrawlerRunConfig())
# Prefetch mode timing
start = time.time()
prefetch_result = await crawler.arun(url, config=CrawlerRunConfig(prefetch=True))
prefetch_time = time.time() - start
# Full processing timing
start = time.time()
full_result = await crawler.arun(url, config=CrawlerRunConfig())
full_time = time.time() - start
print(f"\nTiming comparison:")
print(f" Prefetch mode: {prefetch_time:.3f}s")
print(f" Full processing: {full_time:.3f}s")
print(f" Speedup: {full_time / prefetch_time:.1f}x faster")
print(f"\nOutput comparison:")
print(f" Prefetch - Links found: {len(prefetch_result.links.get('internal', []))}")
print(f" Full - Links found: {len(full_result.links.get('internal', []))}")
print(f" Full - Markdown length: {len(full_result.markdown.raw_markdown) if full_result.markdown else 0}")
async def example_two_phase_crawl():
"""
Example 3: Two-phase crawling pattern.
Phase 1: Fast discovery with prefetch
Phase 2: Full processing on selected URLs
"""
print("\n" + "=" * 60)
print("Example 3: Two-Phase Crawling")
print("=" * 60)
async with AsyncWebCrawler(verbose=False) as crawler:
# ═══════════════════════════════════════════════════════════
# Phase 1: Fast URL discovery
# ═══════════════════════════════════════════════════════════
print("\n--- Phase 1: Fast Discovery ---")
prefetch_config = CrawlerRunConfig(prefetch=True)
start = time.time()
discovery = await crawler.arun("https://books.toscrape.com", config=prefetch_config)
discovery_time = time.time() - start
all_urls = [link["href"] for link in discovery.links.get("internal", [])]
print(f" Discovered {len(all_urls)} URLs in {discovery_time:.2f}s")
# Filter to URLs we care about (e.g., book detail pages)
# On books.toscrape.com, book pages contain "catalogue/" but not "category/"
book_urls = [
url for url in all_urls
if "catalogue/" in url and "category/" not in url
][:5] # Limit to 5 for demo
print(f" Filtered to {len(book_urls)} book pages")
# ═══════════════════════════════════════════════════════════
# Phase 2: Full processing on selected URLs
# ═══════════════════════════════════════════════════════════
print("\n--- Phase 2: Full Processing ---")
full_config = CrawlerRunConfig(
word_count_threshold=10,
remove_overlay_elements=True,
)
results = []
start = time.time()
for url in book_urls:
result = await crawler.arun(url, config=full_config)
if result.success:
results.append(result)
title = result.url.split("/")[-2].replace("-", " ").title()[:40]
md_len = len(result.markdown.raw_markdown) if result.markdown else 0
print(f" Processed: {title}... ({md_len} chars)")
processing_time = time.time() - start
print(f"\n Processed {len(results)} pages in {processing_time:.2f}s")
# ═══════════════════════════════════════════════════════════
# Summary
# ═══════════════════════════════════════════════════════════
print(f"\n--- Summary ---")
print(f" Discovery phase: {discovery_time:.2f}s ({len(all_urls)} URLs)")
print(f" Processing phase: {processing_time:.2f}s ({len(results)} pages)")
print(f" Total time: {discovery_time + processing_time:.2f}s")
print(f" URLs skipped: {len(all_urls) - len(book_urls)} (not matching filter)")
async def example_prefetch_with_deep_crawl():
"""
Example 4: Combine prefetch with deep crawl strategy.
Use prefetch mode during deep crawl for maximum speed.
"""
print("\n" + "=" * 60)
print("Example 4: Prefetch with Deep Crawl")
print("=" * 60)
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
async with AsyncWebCrawler(verbose=False) as crawler:
# Deep crawl with prefetch - maximum discovery speed
config = CrawlerRunConfig(
prefetch=True, # Fast mode
deep_crawl_strategy=BFSDeepCrawlStrategy(
max_depth=1,
max_pages=10,
)
)
print("\nDeep crawling with prefetch mode...")
start = time.time()
result_container = await crawler.arun("https://books.toscrape.com", config=config)
# Handle iterator result from deep crawl
if hasattr(result_container, '__iter__'):
results = list(result_container)
else:
results = [result_container]
elapsed = time.time() - start
# Collect all discovered links
all_internal_links = set()
all_external_links = set()
for result in results:
for link in result.links.get("internal", []):
all_internal_links.add(link["href"])
for link in result.links.get("external", []):
all_external_links.add(link["href"])
print(f"\nResults:")
print(f" Pages crawled: {len(results)}")
print(f" Total internal links discovered: {len(all_internal_links)}")
print(f" Total external links discovered: {len(all_external_links)}")
print(f" Time: {elapsed:.2f}s")
async def example_prefetch_with_raw_html():
"""
Example 5: Prefetch with raw HTML input.
You can also use prefetch mode with raw: URLs for cached content.
"""
print("\n" + "=" * 60)
print("Example 5: Prefetch with Raw HTML")
print("=" * 60)
sample_html = """
Sample Page