Crawl4AI Assistant

import asyncio
import json
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy

async def extract_products():
    # Schema generated from your visual selection
    schema = {
        "name": "Product Catalog",
        "baseSelector": "div.product-card",  # Container you clicked
        "fields": [
            {
                "name": "title",
                "selector": "h3.product-title",
                "type": "text"
            },
            {
                "name": "price",
                "selector": "span.price",
                "type": "text"
            },
            {
                "name": "description",
                "selector": "p.description",
                "type": "text"
            },
            {
                "name": "image",
                "selector": "img.product-image",
                "type": "attribute",
                "attribute": "src"
            }
        ]
    }

    # Create extraction strategy
    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
    
    # Configure the crawler
    config = CrawlerRunConfig(
        extraction_strategy=extraction_strategy
    )

    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(
            url="https://example.com/products",
            config=config
        )
        
        # Parse the extracted data
        products = json.loads(result.extracted_content)
        print(f"Extracted {len(products)} products")
        
        # Display first product
        if products:
            print(json.dumps(products[0], indent=2))
        
        return products

# Run the extraction
if __name__ == "__main__":
    asyncio.run(extract_products())