"""
Shadow DOM Crawling Example
============================

Demonstrates how to use `flatten_shadow_dom=True` to extract content
hidden inside Shadow DOM trees on sites built with Web Components
(Stencil, Lit, Shoelace, Angular Elements, etc.).

Shadow DOM creates encapsulated sub-trees that are invisible to the
normal page serialization (page.content() / outerHTML). The
`flatten_shadow_dom` option walks these trees and produces a single
flat HTML document that includes all shadow content.

This example crawls a Bosch Rexroth product page where the product
description, technical specs, and downloads are rendered entirely
inside Shadow DOM by Stencil.js web components.
"""

import asyncio
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig

URL = "https://store.boschrexroth.com/en/us/p/hydraulic-cylinder-r900999011"


async def main():
    browser_config = BrowserConfig(headless=True)

    # ── 1. Baseline: without shadow DOM flattening ──────────────────
    print("=" * 60)
    print("Without flatten_shadow_dom (baseline)")
    print("=" * 60)

    config = CrawlerRunConfig(
        wait_until="load",
        delay_before_return_html=3.0,
    )

    async with AsyncWebCrawler(config=browser_config) as crawler:
        result = await crawler.arun(URL, config=config)

    md = result.markdown.raw_markdown if result.markdown else ""
    print(f"Markdown length: {len(md)} chars")
    print(f"Has product description: {'mill type design' in md.lower()}")
    print(f"Has technical specs:     {'CDH1' in md}")
    print(f"Has downloads section:   {'Downloads' in md}")
    print()

    # ── 2. With shadow DOM flattening ───────────────────────────────
    print("=" * 60)
    print("With flatten_shadow_dom=True")
    print("=" * 60)

    config = CrawlerRunConfig(
        wait_until="load",
        delay_before_return_html=3.0,
        flatten_shadow_dom=True,
    )

    async with AsyncWebCrawler(config=browser_config) as crawler:
        result = await crawler.arun(URL, config=config)

    md = result.markdown.raw_markdown if result.markdown else ""
    print(f"Markdown length: {len(md)} chars")
    print(f"Has product description: {'mill type design' in md.lower()}")
    print(f"Has technical specs:     {'CDH1' in md}")
    print(f"Has downloads section:   {'Downloads' in md}")
    print()

    # Show the product content section
    idx = md.find("Product Description")
    if idx >= 0:
        print("── Extracted product content ──")
        print(md[idx:idx + 1200])


if __name__ == "__main__":
    asyncio.run(main())