crawl4ai/docs/examples/virtual_scroll_example.py

"""
Example of using the virtual scroll feature to capture content from pages
with virtualized scrolling (like Twitter, Instagram, or other infinite scroll feeds).

This example demonstrates virtual scroll with a local test server serving
different types of scrolling behaviors from HTML files in the assets directory.
"""

import asyncio
import os
import http.server
import socketserver
import threading
from pathlib import Path
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, VirtualScrollConfig, CacheMode, BrowserConfig

# Get the assets directory path
ASSETS_DIR = Path(__file__).parent / "assets"

class TestServer:
    """Simple HTTP server to serve our test HTML files"""
    
    def __init__(self, port=8080):
        self.port = port
        self.httpd = None
        self.server_thread = None
        
    async def start(self):
        """Start the test server"""
        Handler = http.server.SimpleHTTPRequestHandler
        
        # Save current directory and change to assets directory
        self.original_cwd = os.getcwd()
        os.chdir(ASSETS_DIR)
        
        # Try to find an available port
        for _ in range(10):
            try:
                self.httpd = socketserver.TCPServer(("", self.port), Handler)
                break
            except OSError:
                self.port += 1
                
        if self.httpd is None:
            raise RuntimeError("Could not find available port")
            
        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
        self.server_thread.daemon = True
        self.server_thread.start()
        
        # Give server time to start
        await asyncio.sleep(0.5)
        
        print(f"Test server started on http://localhost:{self.port}")
        return self.port
        
    def stop(self):
        """Stop the test server"""
        if self.httpd:
            self.httpd.shutdown()
        # Restore original directory
        if hasattr(self, 'original_cwd'):
            os.chdir(self.original_cwd)
            

async def example_twitter_like_virtual_scroll():
    """
    Example 1: Twitter-like virtual scroll where content is REPLACED.
    This is the classic virtual scroll use case - only visible items exist in DOM.
    """
    print("\n" + "="*60)
    print("EXAMPLE 1: Twitter-like Virtual Scroll")
    print("="*60)
    
    server = TestServer()
    port = await server.start()
    
    try:
        # Configure virtual scroll for Twitter-like timeline
        virtual_config = VirtualScrollConfig(
            container_selector="#timeline",  # The scrollable container
            scroll_count=50,  # Scroll up to 50 times to get all content
            scroll_by="container_height",  # Scroll by container's height
            wait_after_scroll=0.3  # Wait 300ms after each scroll
        )
        
        config = CrawlerRunConfig(
            virtual_scroll_config=virtual_config,
            cache_mode=CacheMode.BYPASS
        )
        
        # TIP: Set headless=False to watch the scrolling happen!
        browser_config = BrowserConfig(
            headless=False,
            viewport={"width": 1280, "height": 800}
        )
        
        async with AsyncWebCrawler(config=browser_config) as crawler:
            result = await crawler.arun(
                url=f"http://localhost:{port}/virtual_scroll_twitter_like.html",
                config=config
            )
            
            # Count tweets captured
            import re
            tweets = re.findall(r'data-tweet-id="(\d+)"', result.html)
            unique_tweets = sorted(set(int(id) for id in tweets))
            
            print(f"\n📊 Results:")
            print(f"   Total HTML length: {len(result.html):,} characters")
            print(f"   Tweets captured: {len(unique_tweets)} unique tweets")
            if unique_tweets:
                print(f"   Tweet IDs range: {min(unique_tweets)} to {max(unique_tweets)}")
                print(f"   Expected range: 0 to 499 (500 tweets total)")
                
                if len(unique_tweets) == 500:
                    print(f"   ✅ SUCCESS! All tweets captured!")
                else:
                    print(f"   ⚠️  Captured {len(unique_tweets)}/500 tweets")
                    
    finally:
        server.stop()


async def example_traditional_append_scroll():
    """
    Example 2: Traditional infinite scroll where content is APPENDED.
    No virtual scroll needed - all content stays in DOM.
    """
    print("\n" + "="*60)
    print("EXAMPLE 2: Traditional Append-Only Scroll")
    print("="*60)
    
    server = TestServer()
    port = await server.start()
    
    try:
        # Configure virtual scroll
        virtual_config = VirtualScrollConfig(
            container_selector=".posts-container",
            scroll_count=15,  # Less scrolls needed since content accumulates
            scroll_by=500,  # Scroll by 500 pixels
            wait_after_scroll=0.4
        )
        
        config = CrawlerRunConfig(
            virtual_scroll_config=virtual_config,
            cache_mode=CacheMode.BYPASS
        )
        
        async with AsyncWebCrawler() as crawler:
            result = await crawler.arun(
                url=f"http://localhost:{port}/virtual_scroll_append_only.html",
                config=config
            )
            
            # Count posts
            import re
            posts = re.findall(r'data-post-id="(\d+)"', result.html)
            unique_posts = sorted(set(int(id) for id in posts))
            
            print(f"\n📊 Results:")
            print(f"   Total HTML length: {len(result.html):,} characters")
            print(f"   Posts captured: {len(unique_posts)} unique posts")
            
            if unique_posts:
                print(f"   Post IDs range: {min(unique_posts)} to {max(unique_posts)}")
                print(f"   ℹ️  Note: This page appends content, so virtual scroll")
                print(f"       just helps trigger more loads. All content stays in DOM.")
                
    finally:
        server.stop()


async def example_instagram_grid():
    """
    Example 3: Instagram-like grid with virtual scroll.
    Grid layout where only visible rows are rendered.
    """
    print("\n" + "="*60)
    print("EXAMPLE 3: Instagram Grid Virtual Scroll")
    print("="*60)
    
    server = TestServer()
    port = await server.start()
    
    try:
        # Configure for grid layout
        virtual_config = VirtualScrollConfig(
            container_selector=".feed-container",  # Container with the grid
            scroll_count=100,  # Many scrolls for 999 posts
            scroll_by="container_height",
            wait_after_scroll=0.2  # Faster scrolling for grid
        )
        
        config = CrawlerRunConfig(
            virtual_scroll_config=virtual_config,
            cache_mode=CacheMode.BYPASS,
            screenshot=True  # Take a screenshot of the final grid
        )
        
        # Show browser for this visual example
        browser_config = BrowserConfig(
            headless=False,
            viewport={"width": 1200, "height": 900}
        )
        
        async with AsyncWebCrawler(config=browser_config) as crawler:
            result = await crawler.arun(
                url=f"http://localhost:{port}/virtual_scroll_instagram_grid.html",
                config=config
            )
            
            # Count posts in grid
            import re
            posts = re.findall(r'data-post-id="(\d+)"', result.html)
            unique_posts = sorted(set(int(id) for id in posts))
            
            print(f"\n📊 Results:")
            print(f"   Posts in grid: {len(unique_posts)} unique posts")
            if unique_posts:
                print(f"   Post IDs range: {min(unique_posts)} to {max(unique_posts)}")
                print(f"   Expected: 0 to 998 (999 posts total)")
                
            # Save screenshot
            if result.screenshot:
                import base64
                with open("instagram_grid_result.png", "wb") as f:
                    f.write(base64.b64decode(result.screenshot))
                print(f"   📸 Screenshot saved as instagram_grid_result.png")
                
    finally:
        server.stop()


async def example_mixed_content():
    """
    Example 4: News feed with mixed behavior.
    Featured articles stay (no virtual scroll), regular articles are virtualized.
    """
    print("\n" + "="*60)
    print("EXAMPLE 4: News Feed with Mixed Behavior")
    print("="*60)
    
    server = TestServer()
    port = await server.start()
    
    try:
        # Configure virtual scroll
        virtual_config = VirtualScrollConfig(
            container_selector="#newsContainer",
            scroll_count=25,
            scroll_by="container_height",
            wait_after_scroll=0.3
        )
        
        config = CrawlerRunConfig(
            virtual_scroll_config=virtual_config,
            cache_mode=CacheMode.BYPASS
        )
        
        async with AsyncWebCrawler() as crawler:
            result = await crawler.arun(
                url=f"http://localhost:{port}/virtual_scroll_news_feed.html",
                config=config
            )
            
            # Count different types of articles
            import re
            featured = re.findall(r'data-article-id="featured-\d+"', result.html)
            regular = re.findall(r'data-article-id="article-(\d+)"', result.html)
            
            print(f"\n📊 Results:")
            print(f"   Featured articles: {len(set(featured))} (always visible)")
            print(f"   Regular articles: {len(set(regular))} unique articles")
            
            if regular:
                regular_ids = sorted(set(int(id) for id in regular))
                print(f"   Regular article IDs: {min(regular_ids)} to {max(regular_ids)}")
                print(f"   ℹ️  Note: Featured articles stay in DOM, only regular")
                print(f"       articles are replaced during virtual scroll")
                
    finally:
        server.stop()


async def compare_with_without_virtual_scroll():
    """
    Comparison: Show the difference between crawling with and without virtual scroll.
    """
    print("\n" + "="*60)
    print("COMPARISON: With vs Without Virtual Scroll")
    print("="*60)
    
    server = TestServer()
    port = await server.start()
    
    try:
        url = f"http://localhost:{port}/virtual_scroll_twitter_like.html"
        
        # First, crawl WITHOUT virtual scroll
        print("\n1️⃣  Crawling WITHOUT virtual scroll...")
        async with AsyncWebCrawler() as crawler:
            config_normal = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
            result_normal = await crawler.arun(url=url, config=config_normal)
            
            # Count items
            import re
            tweets_normal = len(set(re.findall(r'data-tweet-id="(\d+)"', result_normal.html)))
            
        # Then, crawl WITH virtual scroll  
        print("2️⃣  Crawling WITH virtual scroll...")
        virtual_config = VirtualScrollConfig(
            container_selector="#timeline",
            scroll_count=50,
            scroll_by="container_height",
            wait_after_scroll=0.2
        )
        
        config_virtual = CrawlerRunConfig(
            virtual_scroll_config=virtual_config,
            cache_mode=CacheMode.BYPASS
        )
        
        async with AsyncWebCrawler() as crawler:
            result_virtual = await crawler.arun(url=url, config=config_virtual)
            
            # Count items
            tweets_virtual = len(set(re.findall(r'data-tweet-id="(\d+)"', result_virtual.html)))
            
        # Compare results
        print(f"\n📊 Comparison Results:")
        print(f"   Without virtual scroll: {tweets_normal} tweets (only initial visible)")
        print(f"   With virtual scroll: {tweets_virtual} tweets (all content captured)")
        print(f"   Improvement: {tweets_virtual / tweets_normal if tweets_normal > 0 else 'N/A':.1f}x more content!")
        
        print(f"\n   HTML size without: {len(result_normal.html):,} characters")
        print(f"   HTML size with: {len(result_virtual.html):,} characters")
        
    finally:
        server.stop()


if __name__ == "__main__":
    print("""
╔════════════════════════════════════════════════════════════╗
║           Virtual Scroll Examples for Crawl4AI             ║
╚════════════════════════════════════════════════════════════╝

These examples demonstrate different virtual scroll scenarios:
1. Twitter-like (content replaced) - Classic virtual scroll
2. Traditional append - Content accumulates 
3. Instagram grid - Visual grid layout
4. Mixed behavior - Some content stays, some virtualizes

Starting examples...
""")
    
    # Run all examples
    asyncio.run(example_twitter_like_virtual_scroll())
    asyncio.run(example_traditional_append_scroll())
    asyncio.run(example_instagram_grid())
    asyncio.run(example_mixed_content())
    asyncio.run(compare_with_without_virtual_scroll())
    
    print("\n✅ All examples completed!")
    print("\nTIP: Set headless=False in BrowserConfig to watch the scrolling in action!")