fix: #855

feat(deep-crawling): enhance dispatcher to handle multi-page crawl results

Modify MemoryAdaptiveDispatcher to properly handle results from deep crawling operations:
- Add support for processing multiple results from deep crawling
- Implement memory usage distribution across multiple results
- Update task monitoring for deep crawling scenarios
- Modify return types in deep crawling strategies to use RunManyReturn

.github/workflows/main.yml

@@ -1,35 +0,0 @@
-name: Discord GitHub Notifications
-
-on:
-  issues:
-    types: [opened]
-  issue_comment:
-    types: [created]
-  pull_request:
-    types: [opened]
-  discussion:
-    types: [created]
-
-jobs:
-  notify-discord:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set webhook based on event type
-        id: set-webhook
-        run: |
-          if [ "${{ github.event_name }}" == "discussion" ]; then
-            echo "webhook=${{ secrets.DISCORD_DISCUSSIONS_WEBHOOK }}" >> $GITHUB_OUTPUT
-          else
-            echo "webhook=${{ secrets.DISCORD_WEBHOOK }}" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Discord Notification
-        uses: Ilshidur/action-discord@master
-        env:
-          DISCORD_WEBHOOK: ${{ steps.set-webhook.outputs.webhook }}
-        with:
-          args: |
-            ${{ github.event_name == 'issues' && format('📣 New issue created: **{0}** by {1} - {2}', github.event.issue.title, github.event.issue.user.login, github.event.issue.html_url) || 
-            github.event_name == 'issue_comment' && format('💬 New comment on issue **{0}** by {1} - {2}', github.event.issue.title, github.event.comment.user.login, github.event.comment.html_url) ||
-            github.event_name == 'pull_request' && format('🔄 New PR opened: **{0}** by {1} - {2}', github.event.pull_request.title, github.event.pull_request.user.login, github.event.pull_request.html_url) || 
-            format('💬 New discussion started: **{0}** by {1} - {2}', github.event.discussion.title, github.event.discussion.user.login, github.event.discussion.html_url) }}

Dockerfile

@@ -24,7 +24,7 @@ ARG TARGETARCH
 
 LABEL maintainer="unclecode"
 LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
-LABEL version="1.0"
+LABEL version="1.0"    
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
@@ -38,7 +38,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libjpeg-dev \
     redis-server \
     supervisor \
-    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/*
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -63,13 +62,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libcairo2 \
     libasound2 \
     libatspi2.0-0 \
-    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/*
 
 RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \
     apt-get update && apt-get install -y --no-install-recommends \
     nvidia-cuda-toolkit \
-    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/* ; \
 else \
     echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
@@ -79,24 +76,16 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \
     echo "🦾 Installing ARM-specific optimizations"; \
     apt-get update && apt-get install -y --no-install-recommends \
     libopenblas-dev \
-    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/*; \
 elif [ "$TARGETARCH" = "amd64" ]; then \
     echo "🖥️ Installing AMD64-specific optimizations"; \
     apt-get update && apt-get install -y --no-install-recommends \
     libomp-dev \
-    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/*; \
 else \
     echo "Skipping platform-specific optimizations (unsupported platform)"; \
 fi
 
-# Create a non-root user and group
-RUN groupadd -r appuser && useradd --no-log-init -r -g appuser appuser
-
-# Create and set permissions for appuser home directory
-RUN mkdir -p /home/appuser && chown -R appuser:appuser /home/appuser
-
 WORKDIR ${APP_HOME}
 
 RUN echo '#!/bin/bash\n\
@@ -114,7 +103,6 @@ fi' > /tmp/install.sh && chmod +x /tmp/install.sh
 
 COPY . /tmp/project/
 
-# Copy supervisor config first (might need root later, but okay for now)
 COPY deploy/docker/supervisord.conf .
 
 COPY deploy/docker/requirements.txt .
@@ -143,31 +131,16 @@ RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
     else \
         pip install "/tmp/project" ; \
     fi
-
+    
 RUN pip install --no-cache-dir --upgrade pip && \
     /tmp/install.sh && \
     python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
     python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')"
+    
+RUN playwright install --with-deps chromium
 
-RUN crawl4ai-setup
-
-RUN playwright install --with-deps
-
-RUN mkdir -p /home/appuser/.cache/ms-playwright \
-    && cp -r /root/.cache/ms-playwright/chromium-* /home/appuser/.cache/ms-playwright/ \
-    && chown -R appuser:appuser /home/appuser/.cache/ms-playwright
-
-RUN crawl4ai-doctor
-
-# Copy application code
 COPY deploy/docker/* ${APP_HOME}/
 
-# Change ownership of the application directory to the non-root user
-RUN chown -R appuser:appuser ${APP_HOME}
-
-# give permissions to redis persistence dirs if used
-RUN mkdir -p /var/lib/redis /var/log/redis && chown -R appuser:appuser /var/lib/redis /var/log/redis
-
 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
     CMD bash -c '\
     MEM=$(free -m | awk "/^Mem:/{print \$2}"); \
@@ -176,14 +149,8 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
         exit 1; \
     fi && \
     redis-cli ping > /dev/null && \
-    curl -f http://localhost:11235/health || exit 1'
+    curl -f http://localhost:8000/health || exit 1'
 
 EXPOSE 6379
-# Switch to the non-root user before starting the application
-USER appuser
-
-# Set environment variables to ptoduction
-ENV PYTHON_ENV=production 
-
-# Start the application using supervisord
-CMD ["supervisord", "-c", "supervisord.conf"]
+CMD ["supervisord", "-c", "supervisord.conf"]
+    

JOURNAL.md

@@ -1,108 +0,0 @@
-# Development Journal
-
-This journal tracks significant feature additions, bug fixes, and architectural decisions in the crawl4ai project. It serves as both documentation and a historical record of the project's evolution.
-
-## [2025-04-09] Added MHTML Capture Feature
-
-**Feature:** MHTML snapshot capture of crawled pages
-
-**Changes Made:**
-1. Added `capture_mhtml: bool = False` parameter to `CrawlerRunConfig` class
-2. Added `mhtml: Optional[str] = None` field to `CrawlResult` model
-3. Added `mhtml_data: Optional[str] = None` field to `AsyncCrawlResponse` class
-4. Implemented `capture_mhtml()` method in `AsyncPlaywrightCrawlerStrategy` class to capture MHTML via CDP
-5. Modified the crawler to capture MHTML when enabled and pass it to the result
-
-**Implementation Details:**
-- MHTML capture uses Chrome DevTools Protocol (CDP) via Playwright's CDP session API
-- The implementation waits for page to fully load before capturing MHTML content
-- Enhanced waiting for JavaScript content with requestAnimationFrame for better JS content capture
-- We ensure all browser resources are properly cleaned up after capture
-
-**Files Modified:**
-- `crawl4ai/models.py`: Added the mhtml field to CrawlResult
-- `crawl4ai/async_configs.py`: Added capture_mhtml parameter to CrawlerRunConfig
-- `crawl4ai/async_crawler_strategy.py`: Implemented MHTML capture logic
-- `crawl4ai/async_webcrawler.py`: Added mapping from AsyncCrawlResponse.mhtml_data to CrawlResult.mhtml
-
-**Testing:**
-- Created comprehensive tests in `tests/20241401/test_mhtml.py` covering:
-  - Capturing MHTML when enabled
-  - Ensuring mhtml is None when disabled explicitly
-  - Ensuring mhtml is None by default
-  - Capturing MHTML on JavaScript-enabled pages
-
-**Challenges:**
-- Had to improve page loading detection to ensure JavaScript content was fully rendered
-- Tests needed to be run independently due to Playwright browser instance management
-- Modified test expected content to match actual MHTML output
-
-**Why This Feature:**
-The MHTML capture feature allows users to capture complete web pages including all resources (CSS, images, etc.) in a single file. This is valuable for:
-1. Offline viewing of captured pages
-2. Creating permanent snapshots of web content for archival
-3. Ensuring consistent content for later analysis, even if the original site changes
-
-**Future Enhancements to Consider:**
-- Add option to save MHTML to file
-- Support for filtering what resources get included in MHTML
-- Add support for specifying MHTML capture options
-
-## [2025-04-10] Added Network Request and Console Message Capturing
-
-**Feature:** Comprehensive capturing of network requests/responses and browser console messages during crawling
-
-**Changes Made:**
-1. Added `capture_network_requests: bool = False` and `capture_console_messages: bool = False` parameters to `CrawlerRunConfig` class
-2. Added `network_requests: Optional[List[Dict[str, Any]]] = None` and `console_messages: Optional[List[Dict[str, Any]]] = None` fields to both `AsyncCrawlResponse` and `CrawlResult` models
-3. Implemented event listeners in `AsyncPlaywrightCrawlerStrategy._crawl_web()` to capture browser network events and console messages
-4. Added proper event listener cleanup in the finally block to prevent resource leaks
-5. Modified the crawler flow to pass captured data from AsyncCrawlResponse to CrawlResult
-
-**Implementation Details:**
-- Network capture uses Playwright event listeners (`request`, `response`, and `requestfailed`) to record all network activity
-- Console capture uses Playwright event listeners (`console` and `pageerror`) to record console messages and errors
-- Each network event includes metadata like URL, headers, status, and timing information
-- Each console message includes type, text content, and source location when available
-- All captured events include timestamps for chronological analysis
-- Error handling ensures even failed capture attempts won't crash the main crawling process
-
-**Files Modified:**
-- `crawl4ai/models.py`: Added new fields to AsyncCrawlResponse and CrawlResult
-- `crawl4ai/async_configs.py`: Added new configuration parameters to CrawlerRunConfig
-- `crawl4ai/async_crawler_strategy.py`: Implemented capture logic using event listeners
-- `crawl4ai/async_webcrawler.py`: Added data transfer from AsyncCrawlResponse to CrawlResult
-
-**Documentation:**
-- Created detailed documentation in `docs/md_v2/advanced/network-console-capture.md`
-- Added feature to site navigation in `mkdocs.yml`
-- Updated CrawlResult documentation in `docs/md_v2/api/crawl-result.md`
-- Created comprehensive example in `docs/examples/network_console_capture_example.py`
-
-**Testing:**
-- Created `tests/general/test_network_console_capture.py` with tests for:
-  - Verifying capture is disabled by default
-  - Testing network request capturing
-  - Testing console message capturing
-  - Ensuring both capture types can be enabled simultaneously
-  - Checking correct content is captured in expected formats
-
-**Challenges:**
-- Initial implementation had synchronous/asynchronous mismatches in event handlers
-- Needed to fix type of property access vs. method calls in handlers
-- Required careful cleanup of event listeners to prevent memory leaks
-
-**Why This Feature:**
-The network and console capture feature provides deep visibility into web page activity, enabling:
-1. Debugging complex web applications by seeing all network requests and errors
-2. Security analysis to detect unexpected third-party requests and data flows
-3. Performance profiling to identify slow-loading resources
-4. API discovery in single-page applications
-5. Comprehensive analysis of web application behavior
-
-**Future Enhancements to Consider:**
-- Option to filter captured events by type, domain, or content
-- Support for capturing response bodies (with size limits)
-- Aggregate statistics calculation for performance metrics
-- Integration with visualization tools for network waterfall analysis
-- Exporting captures in HAR format for use with external tools

crawl4ai/__version__.py

@@ -1,2 +1,2 @@
 # crawl4ai/_version.py
-__version__ = "0.5.0.post8"
+__version__ = "0.5.0.post4"

crawl4ai/async_configs.py

@@ -15,7 +15,7 @@ from .user_agent_generator import UAGen, ValidUAGenerator  # , OnlineUAGenerator
 from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy
 from .chunking_strategy import ChunkingStrategy, RegexChunking
 
-from .markdown_generation_strategy import MarkdownGenerationStrategy, DefaultMarkdownGenerator
+from .markdown_generation_strategy import MarkdownGenerationStrategy
 from .content_scraping_strategy import ContentScrapingStrategy, WebScrapingStrategy
 from .deep_crawling import DeepCrawlStrategy
 
@@ -29,7 +29,7 @@ from enum import Enum
 
 from .proxy_strategy import ProxyConfig
 try:
-    from .browser.models import DockerConfig
+    from .browser.docker_config import DockerConfig
 except ImportError:
     DockerConfig = None
 
@@ -122,25 +122,23 @@ def from_serializable_dict(data: Any) -> Any:
     # Handle typed data
     if isinstance(data, dict) and "type" in data:
         # Handle plain dictionaries
-        if data["type"] == "dict" and "value" in data:
+        if data["type"] == "dict":
             return {k: from_serializable_dict(v) for k, v in data["value"].items()}
 
         # Import from crawl4ai for class instances
         import crawl4ai
 
-        if hasattr(crawl4ai, data["type"]):
-            cls = getattr(crawl4ai, data["type"])
+        cls = getattr(crawl4ai, data["type"])
 
-            # Handle Enum
-            if issubclass(cls, Enum):
-                return cls(data["params"])
+        # Handle Enum
+        if issubclass(cls, Enum):
+            return cls(data["params"])
 
-            if "params" in data:
-                # Handle class instances
-                constructor_args = {
-                    k: from_serializable_dict(v) for k, v in data["params"].items()
-                }
-                return cls(**constructor_args)
+        # Handle class instances
+        constructor_args = {
+            k: from_serializable_dict(v) for k, v in data["params"].items()
+        }
+        return cls(**constructor_args)
 
     # Handle lists
     if isinstance(data, list):
@@ -178,7 +176,7 @@ class BrowserConfig:
         browser_mode (str): Determines how the browser should be initialized:
                            "builtin" - use the builtin CDP browser running in background
                            "dedicated" - create a new dedicated browser instance each time
-                           "cdp" - use explicit CDP settings provided in cdp_url
+                           "custom" - use explicit CDP settings provided in cdp_url
                            "docker" - run browser in Docker container with isolation
                            Default: "dedicated"
         use_managed_browser (bool): Launch the browser using a managed approach (e.g., via CDP), allowing
@@ -244,7 +242,7 @@ class BrowserConfig:
         channel: str = "chromium",
         proxy: str = None,
         proxy_config: Union[ProxyConfig, dict, None] = None,
-        docker_config: Union[DockerConfig, dict, None] = None,
+        docker_config: Union["DockerConfig", dict, None] = None,
         viewport_width: int = 1080,
         viewport_height: int = 600,
         viewport: dict = None,
@@ -272,7 +270,7 @@ class BrowserConfig:
         host: str = "localhost",
     ):
         self.browser_type = browser_type
-        self.headless = headless or True
+        self.headless = headless
         self.browser_mode = browser_mode
         self.use_managed_browser = use_managed_browser
         self.cdp_url = cdp_url
@@ -291,10 +289,6 @@ class BrowserConfig:
             self.docker_config = DockerConfig.from_kwargs(docker_config)
         else:
             self.docker_config = docker_config
-
-        if self.docker_config:
-            self.user_data_dir = self.docker_config.user_data_dir
-
         self.viewport_width = viewport_width
         self.viewport_height = viewport_height
         self.viewport = viewport
@@ -724,7 +718,7 @@ class CrawlerRunConfig():
         word_count_threshold: int = MIN_WORD_THRESHOLD,
         extraction_strategy: ExtractionStrategy = None,
         chunking_strategy: ChunkingStrategy = RegexChunking(),
-        markdown_generator: MarkdownGenerationStrategy = DefaultMarkdownGenerator(),
+        markdown_generator: MarkdownGenerationStrategy = None,
         only_text: bool = False,
         css_selector: str = None,
         target_elements: List[str] = None,
@@ -774,12 +768,10 @@ class CrawlerRunConfig():
         screenshot_wait_for: float = None,
         screenshot_height_threshold: int = SCREENSHOT_HEIGHT_TRESHOLD,
         pdf: bool = False,
-        capture_mhtml: bool = False,
         image_description_min_word_threshold: int = IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
         image_score_threshold: int = IMAGE_SCORE_THRESHOLD,
         table_score_threshold: int = 7,
         exclude_external_images: bool = False,
-        exclude_all_images: bool = False,
         # Link and Domain Handling Parameters
         exclude_social_media_domains: list = None,
         exclude_external_links: bool = False,
@@ -789,9 +781,6 @@ class CrawlerRunConfig():
         # Debugging and Logging Parameters
         verbose: bool = True,
         log_console: bool = False,
-        # Network and Console Capturing Parameters
-        capture_network_requests: bool = False,
-        capture_console_messages: bool = False,
         # Connection Parameters
         method: str = "GET",
         stream: bool = False,
@@ -867,11 +856,9 @@ class CrawlerRunConfig():
         self.screenshot_wait_for = screenshot_wait_for
         self.screenshot_height_threshold = screenshot_height_threshold
         self.pdf = pdf
-        self.capture_mhtml = capture_mhtml
         self.image_description_min_word_threshold = image_description_min_word_threshold
         self.image_score_threshold = image_score_threshold
         self.exclude_external_images = exclude_external_images
-        self.exclude_all_images = exclude_all_images
         self.table_score_threshold = table_score_threshold
 
         # Link and Domain Handling Parameters
@@ -886,10 +873,6 @@ class CrawlerRunConfig():
         # Debugging and Logging Parameters
         self.verbose = verbose
         self.log_console = log_console
-        
-        # Network and Console Capturing Parameters
-        self.capture_network_requests = capture_network_requests
-        self.capture_console_messages = capture_console_messages
 
         # Connection Parameters
         self.stream = stream
@@ -1004,7 +987,6 @@ class CrawlerRunConfig():
                 "screenshot_height_threshold", SCREENSHOT_HEIGHT_TRESHOLD
             ),
             pdf=kwargs.get("pdf", False),
-            capture_mhtml=kwargs.get("capture_mhtml", False),
             image_description_min_word_threshold=kwargs.get(
                 "image_description_min_word_threshold",
                 IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
@@ -1013,7 +995,6 @@ class CrawlerRunConfig():
                 "image_score_threshold", IMAGE_SCORE_THRESHOLD
             ),
             table_score_threshold=kwargs.get("table_score_threshold", 7),
-            exclude_all_images=kwargs.get("exclude_all_images", False),
             exclude_external_images=kwargs.get("exclude_external_images", False),
             # Link and Domain Handling Parameters
             exclude_social_media_domains=kwargs.get(
@@ -1026,9 +1007,6 @@ class CrawlerRunConfig():
             # Debugging and Logging Parameters
             verbose=kwargs.get("verbose", True),
             log_console=kwargs.get("log_console", False),
-            # Network and Console Capturing Parameters
-            capture_network_requests=kwargs.get("capture_network_requests", False),
-            capture_console_messages=kwargs.get("capture_console_messages", False),
             # Connection Parameters
             method=kwargs.get("method", "GET"),
             stream=kwargs.get("stream", False),
@@ -1106,11 +1084,9 @@ class CrawlerRunConfig():
             "screenshot_wait_for": self.screenshot_wait_for,
             "screenshot_height_threshold": self.screenshot_height_threshold,
             "pdf": self.pdf,
-            "capture_mhtml": self.capture_mhtml,
             "image_description_min_word_threshold": self.image_description_min_word_threshold,
             "image_score_threshold": self.image_score_threshold,
             "table_score_threshold": self.table_score_threshold,
-            "exclude_all_images": self.exclude_all_images,
             "exclude_external_images": self.exclude_external_images,
             "exclude_social_media_domains": self.exclude_social_media_domains,
             "exclude_external_links": self.exclude_external_links,
@@ -1119,8 +1095,6 @@ class CrawlerRunConfig():
             "exclude_internal_links": self.exclude_internal_links,
             "verbose": self.verbose,
             "log_console": self.log_console,
-            "capture_network_requests": self.capture_network_requests,
-            "capture_console_messages": self.capture_console_messages,
             "method": self.method,
             "stream": self.stream,
             "check_robots_txt": self.check_robots_txt,

crawl4ai/async_crawler_strategy.py

@@ -409,11 +409,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
 
         user_agent = kwargs.get("user_agent", self.user_agent)
         # Use browser_manager to get a fresh page & context assigned to this session_id
-        page, context = await self.browser_manager.get_page(CrawlerRunConfig(
-            session_id=session_id,
-            user_agent=user_agent,
-            **kwargs,
-        ))
+        page, context = await self.browser_manager.get_page(session_id, user_agent)
         return session_id
 
     async def crawl(
@@ -451,17 +447,12 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
                 html = f.read()
             if config.screenshot:
                 screenshot_data = await self._generate_screenshot_from_html(html)
-            if config.capture_console_messages:
-                page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
-                captured_console = await self._capture_console_messages(page, url)
-
             return AsyncCrawlResponse(
                 html=html,
                 response_headers=response_headers,
                 status_code=status_code,
                 screenshot=screenshot_data,
                 get_delayed_content=None,
-                console_messages=captured_console,
             )
 
         elif url.startswith("raw:") or url.startswith("raw://"):
@@ -487,7 +478,6 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
     ) -> AsyncCrawlResponse:
         """
         Internal method to crawl web URLs with the specified configuration.
-        Includes optional network and console capturing.
 
         Args:
             url (str): The web URL to crawl
@@ -504,10 +494,6 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
 
         # Reset downloaded files list for new crawl
         self._downloaded_files = []
-        
-        # Initialize capture lists
-        captured_requests = []
-        captured_console = []
 
         # Handle user agent with magic mode
         user_agent_to_override = config.user_agent
@@ -535,144 +521,9 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
         # Call hook after page creation
         await self.execute_hook("on_page_context_created", page, context=context, config=config)
 
-        # Network Request Capturing
-        if config.capture_network_requests:
-            async def handle_request_capture(request):
-                try:
-                    post_data_str = None
-                    try:
-                        # Be cautious with large post data
-                        post_data = request.post_data_buffer
-                        if post_data:
-                             # Attempt to decode, fallback to base64 or size indication
-                             try:
-                                 post_data_str = post_data.decode('utf-8', errors='replace')
-                             except UnicodeDecodeError:
-                                 post_data_str = f"[Binary data: {len(post_data)} bytes]"
-                    except Exception:
-                        post_data_str = "[Error retrieving post data]"
-
-                    captured_requests.append({
-                        "event_type": "request",
-                        "url": request.url,
-                        "method": request.method,
-                        "headers": dict(request.headers), # Convert Header dict
-                        "post_data": post_data_str,
-                        "resource_type": request.resource_type,
-                        "is_navigation_request": request.is_navigation_request(),
-                        "timestamp": time.time()
-                    })
-                except Exception as e:
-                    if self.logger:
-                        self.logger.warning(f"Error capturing request details for {request.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "request_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()})
-
-            async def handle_response_capture(response):
-                try:
-                    captured_requests.append({
-                        "event_type": "response",
-                        "url": response.url,
-                        "status": response.status,
-                        "status_text": response.status_text,
-                        "headers": dict(response.headers), # Convert Header dict
-                        "from_service_worker": response.from_service_worker,
-                        "request_timing": response.request.timing, # Detailed timing info
-                        "timestamp": time.time()
-                    })
-                except Exception as e:
-                    if self.logger:
-                        self.logger.warning(f"Error capturing response details for {response.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "response_capture_error", "url": response.url, "error": str(e), "timestamp": time.time()})
-
-            async def handle_request_failed_capture(request):
-                 try:
-                    captured_requests.append({
-                        "event_type": "request_failed",
-                        "url": request.url,
-                        "method": request.method,
-                        "resource_type": request.resource_type,
-                        "failure_text": str(request.failure) if request.failure else "Unknown failure",
-                        "timestamp": time.time()
-                    })
-                 except Exception as e:
-                    if self.logger:
-                        self.logger.warning(f"Error capturing request failed details for {request.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "request_failed_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()})
-
-            page.on("request", handle_request_capture)
-            page.on("response", handle_response_capture)
-            page.on("requestfailed", handle_request_failed_capture)
-
-        # Console Message Capturing
-        if config.capture_console_messages:
-            def handle_console_capture(msg):
-                try:
-                    message_type = "unknown"
-                    try:
-                        message_type = msg.type
-                    except:
-                        pass
-                        
-                    message_text = "unknown"
-                    try:
-                        message_text = msg.text
-                    except:
-                        pass
-                        
-                    # Basic console message with minimal content
-                    entry = {
-                        "type": message_type,
-                        "text": message_text,
-                        "timestamp": time.time()
-                    }
-                    
-                    captured_console.append(entry)
-                    
-                except Exception as e:
-                    if self.logger:
-                        self.logger.warning(f"Error capturing console message: {e}", tag="CAPTURE")
-                    # Still add something to the list even on error
-                    captured_console.append({
-                        "type": "console_capture_error", 
-                        "error": str(e), 
-                        "timestamp": time.time()
-                    })
-
-            def handle_pageerror_capture(err):
-                try:
-                    error_message = "Unknown error"
-                    try:
-                        error_message = err.message
-                    except:
-                        pass
-                        
-                    error_stack = ""
-                    try:
-                        error_stack = err.stack
-                    except:
-                        pass
-                        
-                    captured_console.append({
-                        "type": "error",
-                        "text": error_message,
-                        "stack": error_stack,
-                        "timestamp": time.time()
-                    })
-                except Exception as e:
-                    if self.logger:
-                        self.logger.warning(f"Error capturing page error: {e}", tag="CAPTURE")
-                    captured_console.append({
-                        "type": "pageerror_capture_error", 
-                        "error": str(e), 
-                        "timestamp": time.time()
-                    })
-
-            # Add event listeners directly
-            page.on("console", handle_console_capture)
-            page.on("pageerror", handle_pageerror_capture)
-
         # Set up console logging if requested
         if config.log_console:
+
             def log_consol(
                 msg, console_log_type="debug"
             ):  # Corrected the parameter syntax
@@ -985,18 +836,14 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
                 "before_return_html", page=page, html=html, context=context, config=config
             )
 
-            # Handle PDF, MHTML and screenshot generation
+            # Handle PDF and screenshot generation
             start_export_time = time.perf_counter()
             pdf_data = None
             screenshot_data = None
-            mhtml_data = None
 
             if config.pdf:
                 pdf_data = await self.export_pdf(page)
 
-            if config.capture_mhtml:
-                mhtml_data = await self.capture_mhtml(page)
-
             if config.screenshot:
                 if config.screenshot_wait_for:
                     await asyncio.sleep(config.screenshot_wait_for)
@@ -1004,9 +851,9 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
                     page, screenshot_height_threshold=config.screenshot_height_threshold
                 )
 
-            if screenshot_data or pdf_data or mhtml_data:
+            if screenshot_data or pdf_data:
                 self.logger.info(
-                    message="Exporting media (PDF/MHTML/screenshot) took {duration:.2f}s",
+                    message="Exporting PDF and taking screenshot took {duration:.2f}s",
                     tag="EXPORT",
                     params={"duration": time.perf_counter() - start_export_time},
                 )
@@ -1029,16 +876,12 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
                 status_code=status_code,
                 screenshot=screenshot_data,
                 pdf_data=pdf_data,
-                mhtml_data=mhtml_data,
                 get_delayed_content=get_delayed_content,
                 ssl_certificate=ssl_cert,
                 downloaded_files=(
                     self._downloaded_files if self._downloaded_files else None
                 ),
                 redirected_url=redirected_url,
-                # Include captured data if enabled
-                network_requests=captured_requests if config.capture_network_requests else None,
-                console_messages=captured_console if config.capture_console_messages else None,
             )
 
         except Exception as e:
@@ -1047,15 +890,6 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
         finally:
             # If no session_id is given we should close the page
             if not config.session_id:
-                # Detach listeners before closing to prevent potential errors during close
-                if config.capture_network_requests:
-                    page.remove_listener("request", handle_request_capture)
-                    page.remove_listener("response", handle_response_capture)
-                    page.remove_listener("requestfailed", handle_request_failed_capture)
-                if config.capture_console_messages:
-                    page.remove_listener("console", handle_console_capture)
-                    page.remove_listener("pageerror", handle_pageerror_capture)
-                
                 await page.close()
 
     async def _handle_full_page_scan(self, page: Page, scroll_delay: float = 0.1):
@@ -1218,107 +1052,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
         """
         pdf_data = await page.pdf(print_background=True)
         return pdf_data
-        
-    async def capture_mhtml(self, page: Page) -> Optional[str]:
-        """
-        Captures the current page as MHTML using CDP.
-        
-        MHTML (MIME HTML) is a web page archive format that combines the HTML content 
-        with its resources (images, CSS, etc.) into a single MIME-encoded file.
-        
-        Args:
-            page (Page): The Playwright page object
-            
-        Returns:
-            Optional[str]: The MHTML content as a string, or None if there was an error
-        """
-        try:
-            # Ensure the page is fully loaded before capturing
-            try:
-                # Wait for DOM content and network to be idle
-                await page.wait_for_load_state("domcontentloaded", timeout=5000)
-                await page.wait_for_load_state("networkidle", timeout=5000)
-                
-                # Give a little extra time for JavaScript execution
-                await page.wait_for_timeout(1000)
-                
-                # Wait for any animations to complete
-                await page.evaluate("""
-                    () => new Promise(resolve => {
-                        // First requestAnimationFrame gets scheduled after the next repaint
-                        requestAnimationFrame(() => {
-                            // Second requestAnimationFrame gets called after all animations complete
-                            requestAnimationFrame(resolve);
-                        });
-                    })
-                """)
-            except Error as e:
-                if self.logger:
-                    self.logger.warning(
-                        message="Wait for load state timed out: {error}",
-                        tag="MHTML",
-                        params={"error": str(e)},
-                    )
-            
-            # Create a new CDP session
-            cdp_session = await page.context.new_cdp_session(page)
-            
-            # Call Page.captureSnapshot with format "mhtml"
-            result = await cdp_session.send("Page.captureSnapshot", {"format": "mhtml"})
-            
-            # The result contains a 'data' field with the MHTML content
-            mhtml_content = result.get("data")
-            
-            # Detach the CDP session to clean up resources
-            await cdp_session.detach()
-            
-            return mhtml_content
-        except Exception as e:
-            # Log the error but don't raise it - we'll just return None for the MHTML
-            if self.logger:
-                self.logger.error(
-                    message="Failed to capture MHTML: {error}",
-                    tag="MHTML",
-                    params={"error": str(e)},
-                )
-            return None
 
-    async def _capture_console_messages(
-        self, page: Page, file_path: str
-    ) -> List[Dict[str, Union[str, float]]]:
-        """
-        Captures console messages from the page.
-        Args:
-
-            page (Page): The Playwright page object
-        Returns:
-            List[Dict[str, Union[str, float]]]: A list of captured console messages
-        """
-        captured_console = []
-
-        def handle_console_message(msg):
-            try:
-                message_type = msg.type
-                message_text = msg.text
-
-                entry = {
-                    "type": message_type,
-                    "text": message_text,
-                    "timestamp": time.time(),
-                }
-                captured_console.append(entry)
-            except Exception as e:
-                if self.logger:
-                    self.logger.warning(
-                        f"Error capturing console message: {e}", tag="CAPTURE"
-                    )
-
-        page.on("console", handle_console_message)
-        
-        await page.goto(file_path)
-
-        return captured_console
-        
     async def take_screenshot(self, page, **kwargs) -> str:
         """
         Take a screenshot of the current page.

crawl4ai/async_dispatcher.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional, List, Tuple
+from typing import Dict, Optional, List, Tuple, Union
 from .async_configs import CrawlerRunConfig
 from .models import (
     CrawlResult,
@@ -183,7 +183,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
         config: CrawlerRunConfig,
         task_id: str,
         retry_count: int = 0,
-    ) -> CrawlerTaskResult:
+    ) -> Union[CrawlerTaskResult, List[CrawlerTaskResult]]:
         start_time = time.time()
         error_message = ""
         memory_usage = peak_memory = 0.0
@@ -244,8 +244,53 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             end_memory = process.memory_info().rss / (1024 * 1024)
             memory_usage = peak_memory = end_memory - start_memory
             
-            # Handle rate limiting
-            if self.rate_limiter and result.status_code:
+            # Check if we have a container with multiple results (deep crawl result)
+            if isinstance(result, list) or (hasattr(result, '_results') and len(result._results) > 1):
+                # Handle deep crawling results - create a list of task results
+                task_results = []
+                result_list = result if isinstance(result, list) else result._results
+                
+                for idx, single_result in enumerate(result_list):
+                    # Create individual task result for each crawled page
+                    sub_task_id = f"{task_id}_{idx}"
+                    single_memory = memory_usage / len(result_list)  # Distribute memory usage
+                    
+                    # Only update rate limiter for first result which corresponds to the original URL
+                    if idx == 0 and self.rate_limiter and hasattr(single_result, 'status_code') and single_result.status_code:
+                        if not self.rate_limiter.update_delay(url, single_result.status_code):
+                            error_msg = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
+                            if self.monitor:
+                                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                    
+                    task_result = CrawlerTaskResult(
+                        task_id=sub_task_id,
+                        url=single_result.url,
+                        result=single_result,
+                        memory_usage=single_memory,
+                        peak_memory=single_memory,
+                        start_time=start_time,
+                        end_time=time.time(),
+                        error_message=single_result.error_message if not single_result.success else "",
+                        retry_count=retry_count
+                    )
+                    task_results.append(task_result)
+                
+                # Update monitor with completion status based on the first/primary result
+                if self.monitor:
+                    primary_result = result_list[0]
+                    if not primary_result.success:
+                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                    else:
+                        self.monitor.update_task(
+                            task_id, 
+                            status=CrawlStatus.COMPLETED,
+                            extra_info=f"Deep crawl: {len(result_list)} pages"
+                        )
+                
+                return task_results
+            
+            # Handle single result (original behavior)
+            if self.rate_limiter and hasattr(result, 'status_code') and result.status_code:
                 if not self.rate_limiter.update_delay(url, result.status_code):
                     error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
                     if self.monitor:
@@ -291,7 +336,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             error_message=error_message,
             retry_count=retry_count
         )
-        
+            
     async def run_urls(
         self,
         urls: List[str],
@@ -356,8 +401,13 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
                     
                     # Process completed tasks
                     for completed_task in done:
-                        result = await completed_task
-                        results.append(result)
+                        task_result = await completed_task
+                        
+                        # Handle both single results and lists of results
+                        if isinstance(task_result, list):
+                            results.extend(task_result)
+                        else:
+                            results.append(task_result)
                         
                     # Update active tasks list
                     active_tasks = list(pending)
@@ -379,7 +429,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             memory_monitor.cancel()
             if self.monitor:
                 self.monitor.stop()
-                
+
     async def _update_queue_priorities(self):
         """Periodically update priorities of items in the queue to prevent starvation"""
         # Skip if queue is empty

crawl4ai/async_logger.py

@@ -156,22 +156,9 @@ class AsyncLogger(AsyncLoggerBase):
                 formatted_message = message.format(**params)
 
                 # Then apply colors if specified
-                color_map = {
-                    "green": Fore.GREEN,
-                    "red": Fore.RED,
-                    "yellow": Fore.YELLOW,
-                    "blue": Fore.BLUE,
-                    "cyan": Fore.CYAN,
-                    "magenta": Fore.MAGENTA,
-                    "white": Fore.WHITE,
-                    "black": Fore.BLACK,
-                    "reset": Style.RESET_ALL,
-                }
                 if colors:
                     for key, color in colors.items():
                         # Find the formatted value in the message and wrap it with color
-                        if color in color_map:
-                            color = color_map[color]
                         if key in params:
                             value_str = str(params[key])
                             formatted_message = formatted_message.replace(

crawl4ai/async_webcrawler.py

@@ -4,25 +4,18 @@ import sys
 import time
 from colorama import Fore
 from pathlib import Path
-from typing import Optional, List
+from typing import Optional, List, Generic, TypeVar
 import json
 import asyncio
 
 # from contextlib import nullcontext, asynccontextmanager
 from contextlib import asynccontextmanager
-from .models import (
-    CrawlResult,
-    MarkdownGenerationResult,
-    DispatchResult,
-    ScrapingResult,
-    CrawlResultContainer,
-    RunManyReturn
-)
+from .models import CrawlResult, MarkdownGenerationResult, DispatchResult, ScrapingResult
 from .async_database import async_db_manager
 from .chunking_strategy import *  # noqa: F403
 from .chunking_strategy import IdentityChunking
 from .content_filter_strategy import *  # noqa: F403
-from .extraction_strategy import *  # noqa: F403
+from .extraction_strategy import * # noqa: F403
 from .extraction_strategy import NoExtractionStrategy
 from .async_crawler_strategy import (
     AsyncCrawlerStrategy,
@@ -37,7 +30,7 @@ from .markdown_generation_strategy import (
 from .deep_crawling import DeepCrawlDecorator
 from .async_logger import AsyncLogger, AsyncLoggerBase
 from .async_configs import BrowserConfig, CrawlerRunConfig
-from .async_dispatcher import *  # noqa: F403
+from .async_dispatcher import * # noqa: F403
 from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter
 
 from .utils import (
@@ -49,6 +42,45 @@ from .utils import (
     RobotsParser,
 )
 
+from typing import Union, AsyncGenerator
+
+CrawlResultT = TypeVar('CrawlResultT', bound=CrawlResult)
+# RunManyReturn = Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+
+class CrawlResultContainer(Generic[CrawlResultT]):
+    def __init__(self, results: Union[CrawlResultT, List[CrawlResultT]]):
+        # Normalize to a list
+        if isinstance(results, list):
+            self._results = results
+        else:
+            self._results = [results]
+
+    def __iter__(self):
+        return iter(self._results)
+
+    def __getitem__(self, index):
+        return self._results[index]
+
+    def __len__(self):
+        return len(self._results)
+
+    def __getattr__(self, attr):
+        # Delegate attribute access to the first element.
+        if self._results:
+            return getattr(self._results[0], attr)
+        raise AttributeError(f"{self.__class__.__name__} object has no attribute '{attr}'")
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._results!r})"
+
+# Redefine the union type. Now synchronous calls always return a container,
+# while stream mode is handled with an AsyncGenerator.
+RunManyReturn = Union[
+    CrawlResultContainer[CrawlResultT],
+    AsyncGenerator[CrawlResultT, None]
+]
+
+
 
 class AsyncWebCrawler:
     """
@@ -161,18 +193,45 @@ class AsyncWebCrawler:
 
         # Decorate arun method with deep crawling capabilities
         self._deep_handler = DeepCrawlDecorator(self)
-        self.arun = self._deep_handler(self.arun)
+        self.arun = self._deep_handler(self.arun)  
 
     async def start(self):
         """
         Start the crawler explicitly without using context manager.
         This is equivalent to using 'async with' but gives more control over the lifecycle.
+
+        This method will:
+        1. Check for builtin browser if browser_mode is 'builtin'
+        2. Initialize the browser and context
+        3. Perform warmup sequence
+        4. Return the crawler instance for method chaining
+
         Returns:
             AsyncWebCrawler: The initialized crawler instance
         """
+        # Check for builtin browser if requested
+        if self.browser_config.browser_mode == "builtin" and not self.browser_config.cdp_url:
+            # Import here to avoid circular imports
+            from .browser_profiler import BrowserProfiler
+            profiler = BrowserProfiler(logger=self.logger)
+            
+            # Get builtin browser info or launch if needed
+            browser_info = profiler.get_builtin_browser_info()
+            if not browser_info:
+                self.logger.info("Builtin browser not found, launching new instance...", tag="BROWSER")
+                cdp_url = await profiler.launch_builtin_browser()
+                if not cdp_url:
+                    self.logger.warning("Failed to launch builtin browser, falling back to dedicated browser", tag="BROWSER")
+                else:
+                    self.browser_config.cdp_url = cdp_url
+                    self.browser_config.use_managed_browser = True
+            else:
+                self.logger.info(f"Using existing builtin browser at {browser_info.get('cdp_url')}", tag="BROWSER")
+                self.browser_config.cdp_url = browser_info.get('cdp_url')
+                self.browser_config.use_managed_browser = True
+                
         await self.crawler_strategy.__aenter__()
-        self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT")
-        self.ready = True
+        await self.awarmup()
         return self
 
     async def close(self):
@@ -192,6 +251,18 @@ class AsyncWebCrawler:
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         await self.close()
 
+    async def awarmup(self):
+        """
+        Initialize the crawler with warm-up sequence.
+
+        This method:
+        1. Logs initialization info
+        2. Sets up browser configuration
+        3. Marks the crawler as ready
+        """
+        self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT")
+        self.ready = True
+
     @asynccontextmanager
     async def nullcontext(self):
         """异步空上下文管理器"""
@@ -234,7 +305,7 @@ class AsyncWebCrawler:
         # Auto-start if not ready
         if not self.ready:
             await self.start()
-
+            
         config = config or CrawlerRunConfig()
         if not isinstance(url, str) or not url:
             raise ValueError("Invalid URL, make sure the URL is a non-empty string")
@@ -248,7 +319,9 @@ class AsyncWebCrawler:
                     config.cache_mode = CacheMode.ENABLED
 
                 # Create cache context
-                cache_context = CacheContext(url, config.cache_mode, False)
+                cache_context = CacheContext(
+                    url, config.cache_mode, False
+                )
 
                 # Initialize processing variables
                 async_response: AsyncCrawlResponse = None
@@ -278,7 +351,7 @@ class AsyncWebCrawler:
                     # if config.screenshot and not screenshot or config.pdf and not pdf:
                     if config.screenshot and not screenshot_data:
                         cached_result = None
-
+                    
                     if config.pdf and not pdf_data:
                         cached_result = None
 
@@ -310,18 +383,14 @@ class AsyncWebCrawler:
 
                     # Check robots.txt if enabled
                     if config and config.check_robots_txt:
-                        if not await self.robots_parser.can_fetch(
-                            url, self.browser_config.user_agent
-                        ):
+                        if not await self.robots_parser.can_fetch(url, self.browser_config.user_agent):
                             return CrawlResult(
                                 url=url,
                                 html="",
                                 success=False,
                                 status_code=403,
                                 error_message="Access denied by robots.txt",
-                                response_headers={
-                                    "X-Robots-Status": "Blocked by robots.txt"
-                                },
+                                response_headers={"X-Robots-Status": "Blocked by robots.txt"}
                             )
 
                     ##############################
@@ -348,7 +417,7 @@ class AsyncWebCrawler:
                     ###############################################################
                     # Process the HTML content, Call CrawlerStrategy.process_html #
                     ###############################################################
-                    crawl_result: CrawlResult = await self.aprocess_html(
+                    crawl_result : CrawlResult = await self.aprocess_html(
                         url=url,
                         html=html,
                         extracted_content=extracted_content,
@@ -365,11 +434,9 @@ class AsyncWebCrawler:
                     crawl_result.response_headers = async_response.response_headers
                     crawl_result.downloaded_files = async_response.downloaded_files
                     crawl_result.js_execution_result = js_execution_result
-                    crawl_result.mhtml = async_response.mhtml_data
-                    crawl_result.ssl_certificate = async_response.ssl_certificate
-                    # Add captured network and console data if available
-                    crawl_result.network_requests = async_response.network_requests
-                    crawl_result.console_messages = async_response.console_messages
+                    crawl_result.ssl_certificate = (
+                        async_response.ssl_certificate
+                    )  # Add SSL certificate
 
                     crawl_result.success = bool(html)
                     crawl_result.session_id = getattr(config, "session_id", None)
@@ -427,7 +494,7 @@ class AsyncWebCrawler:
                     tag="ERROR",
                 )
 
-                return CrawlResultContainer(
+                return  CrawlResultContainer(
                     CrawlResult(
                         url=url, html="", success=False, error_message=error_message
                     )
@@ -472,14 +539,15 @@ class AsyncWebCrawler:
 
             # Process HTML content
             params = config.__dict__.copy()
-            params.pop("url", None)
+            params.pop("url", None)            
             # add keys from kwargs to params that doesn't exist in params
             params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
 
+            
             ################################
             # Scraping Strategy Execution  #
             ################################
-            result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
+            result : ScrapingResult = scraping_strategy.scrap(url, html, **params)
 
             if result is None:
                 raise ValueError(
@@ -528,10 +596,7 @@ class AsyncWebCrawler:
         self.logger.info(
             message="{url:.50}... | Time: {timing}s",
             tag="SCRAPE",
-            params={
-                "url": _url,
-                "timing": int((time.perf_counter() - t1) * 1000) / 1000,
-            },
+            params={"url": _url, "timing": int((time.perf_counter() - t1) * 1000) / 1000},
         )
 
         ################################
@@ -606,22 +671,10 @@ class AsyncWebCrawler:
     async def arun_many(
         self,
         urls: List[str],
-        config: Optional[CrawlerRunConfig] = None,
+        config: Optional[CrawlerRunConfig] = None, 
         dispatcher: Optional[BaseDispatcher] = None,
-        # Legacy parameters maintained for backwards compatibility
-        # word_count_threshold=MIN_WORD_THRESHOLD,
-        # extraction_strategy: ExtractionStrategy = None,
-        # chunking_strategy: ChunkingStrategy = RegexChunking(),
-        # content_filter: RelevantContentFilter = None,
-        # cache_mode: Optional[CacheMode] = None,
-        # bypass_cache: bool = False,
-        # css_selector: str = None,
-        # screenshot: bool = False,
-        # pdf: bool = False,
-        # user_agent: str = None,
-        # verbose=True,
-        **kwargs,
-    ) -> RunManyReturn:
+        **kwargs
+        ) -> RunManyReturn:
         """
         Runs the crawler for multiple URLs concurrently using a configurable dispatcher strategy.
 
@@ -653,20 +706,7 @@ class AsyncWebCrawler:
             print(f"Processed {result.url}: {len(result.markdown)} chars")
         """
         config = config or CrawlerRunConfig()
-        # if config is None:
-        #     config = CrawlerRunConfig(
-        #         word_count_threshold=word_count_threshold,
-        #         extraction_strategy=extraction_strategy,
-        #         chunking_strategy=chunking_strategy,
-        #         content_filter=content_filter,
-        #         cache_mode=cache_mode,
-        #         bypass_cache=bypass_cache,
-        #         css_selector=css_selector,
-        #         screenshot=screenshot,
-        #         pdf=pdf,
-        #         verbose=verbose,
-        #         **kwargs,
-        #     )
+
 
         if dispatcher is None:
             dispatcher = MemoryAdaptiveDispatcher(
@@ -677,32 +717,37 @@ class AsyncWebCrawler:
 
         def transform_result(task_result):
             return (
-                setattr(
-                    task_result.result,
-                    "dispatch_result",
-                    DispatchResult(
-                        task_id=task_result.task_id,
-                        memory_usage=task_result.memory_usage,
-                        peak_memory=task_result.peak_memory,
-                        start_time=task_result.start_time,
-                        end_time=task_result.end_time,
-                        error_message=task_result.error_message,
-                    ),
+                    setattr(task_result.result, 'dispatch_result', 
+                        DispatchResult(
+                            task_id=task_result.task_id,
+                            memory_usage=task_result.memory_usage,
+                            peak_memory=task_result.peak_memory,
+                            start_time=task_result.start_time,
+                            end_time=task_result.end_time,
+                            error_message=task_result.error_message,
+                        )
+                    ) or task_result.result
                 )
-                or task_result.result
-            )
 
         stream = config.stream
-
+        
         if stream:
-
             async def result_transformer():
-                async for task_result in dispatcher.run_urls_stream(
-                    crawler=self, urls=urls, config=config
-                ):
+                async for task_result in dispatcher.run_urls_stream(crawler=self, urls=urls, config=config):
                     yield transform_result(task_result)
-
             return result_transformer()
         else:
             _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config)
-            return [transform_result(res) for res in _results]
+            return [transform_result(res) for res in _results]    
+
+    async def aclear_cache(self):
+        """Clear the cache database."""
+        await async_db_manager.cleanup()
+
+    async def aflush_cache(self):
+        """Flush the cache database."""
+        await async_db_manager.aflush_db()
+
+    async def aget_cache_size(self):
+        """Get the total number of cached items."""
+        return await async_db_manager.aget_total_count()

crawl4ai/browser/__init__.py

@@ -0,0 +1,10 @@
+"""Browser management module for Crawl4AI.
+
+This module provides browser management capabilities using different strategies
+for browser creation and interaction.
+"""
+
+from .manager import BrowserManager
+from .profiles import BrowserProfileManager
+
+__all__ = ['BrowserManager', 'BrowserProfileManager']

crawl4ai/browser/docker/connect.Dockerfile

@@ -0,0 +1,61 @@
+FROM ubuntu:22.04
+
+# Install dependencies with comprehensive Chromium support
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    gnupg \
+    ca-certificates \
+    fonts-liberation \
+    # Sound support
+    libasound2 \
+    # Accessibility support
+    libatspi2.0-0 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    # Graphics and rendering
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    # X11 and window system
+    libx11-6 \
+    libxcb1 \
+    libxkbcommon0 \
+    # Text and internationalization
+    libpango-1.0-0 \
+    libcairo2 \
+    # Printing support
+    libcups2 \
+    # System libraries
+    libdbus-1-3 \
+    libnss3 \
+    libnspr4 \
+    libglib2.0-0 \
+    # Utilities
+    xdg-utils \
+    socat \
+    # Process management
+    procps \
+    # Clean up
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
+    apt-get update && \
+    apt-get install -y google-chrome-stable && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create data directory for user data
+RUN mkdir -p /data && chmod 777 /data
+
+# Add a startup script
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Set entrypoint
+ENTRYPOINT ["/start.sh"]

crawl4ai/browser/docker/launch.Dockerfile

@@ -0,0 +1,57 @@
+FROM ubuntu:22.04
+
+# Install dependencies with comprehensive Chromium support
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    gnupg \
+    ca-certificates \
+    fonts-liberation \
+    # Sound support
+    libasound2 \
+    # Accessibility support
+    libatspi2.0-0 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    # Graphics and rendering
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    # X11 and window system
+    libx11-6 \
+    libxcb1 \
+    libxkbcommon0 \
+    # Text and internationalization
+    libpango-1.0-0 \
+    libcairo2 \
+    # Printing support
+    libcups2 \
+    # System libraries
+    libdbus-1-3 \
+    libnss3 \
+    libnspr4 \
+    libglib2.0-0 \
+    # Utilities
+    xdg-utils \
+    socat \
+    # Process management
+    procps \
+    # Clean up
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Chrome
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
+    apt-get update && \
+    apt-get install -y google-chrome-stable && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create data directory for user data
+RUN mkdir -p /data && chmod 777 /data
+
+# Keep container running without starting Chrome
+CMD ["tail", "-f", "/dev/null"]

crawl4ai/browser/docker_config.py

@@ -0,0 +1,133 @@
+"""Docker configuration module for Crawl4AI browser automation.
+
+This module provides configuration classes for Docker-based browser automation,
+allowing flexible configuration of Docker containers for browsing.
+"""
+
+from typing import Dict, List, Optional, Union
+
+
+class DockerConfig:
+    """Configuration for Docker-based browser automation.
+    
+    This class contains Docker-specific settings to avoid cluttering BrowserConfig.
+    
+    Attributes:
+        mode (str): Docker operation mode - "connect" or "launch".
+            - "connect": Uses a container with Chrome already running
+            - "launch": Dynamically configures and starts Chrome in container
+        image (str): Docker image to use. If None, defaults from DockerUtils are used.
+        registry_file (str): Path to container registry file for persistence.
+        persistent (bool): Keep container running after browser closes.
+        remove_on_exit (bool): Remove container on exit when not persistent.
+        network (str): Docker network to use.
+        volumes (List[str]): Volume mappings (e.g., ["host_path:container_path"]).
+        env_vars (Dict[str, str]): Environment variables to set in container.
+        extra_args (List[str]): Additional docker run arguments.
+        host_port (int): Host port to map to container's 9223 port.
+        user_data_dir (str): Path to user data directory on host.
+        container_user_data_dir (str): Path to user data directory in container.
+    """
+    
+    def __init__(
+        self,
+        mode: str = "connect",                     # "connect" or "launch" 
+        image: Optional[str] = None,               # Docker image to use
+        registry_file: Optional[str] = None,       # Path to registry file
+        persistent: bool = False,                  # Keep container running after browser closes
+        remove_on_exit: bool = True,               # Remove container on exit when not persistent
+        network: Optional[str] = None,             # Docker network to use
+        volumes: List[str] = None,                 # Volume mappings
+        env_vars: Dict[str, str] = None,           # Environment variables
+        extra_args: List[str] = None,              # Additional docker run arguments
+        host_port: Optional[int] = None,           # Host port to map to container's 9223
+        user_data_dir: Optional[str] = None,       # Path to user data directory on host
+        container_user_data_dir: str = "/data",    # Path to user data directory in container
+    ):
+        """Initialize Docker configuration.
+        
+        Args:
+            mode: Docker operation mode ("connect" or "launch")
+            image: Docker image to use
+            registry_file: Path to container registry file
+            persistent: Whether to keep container running after browser closes
+            remove_on_exit: Whether to remove container on exit when not persistent
+            network: Docker network to use
+            volumes: Volume mappings as list of strings
+            env_vars: Environment variables as dictionary
+            extra_args: Additional docker run arguments
+            host_port: Host port to map to container's 9223
+            user_data_dir: Path to user data directory on host
+            container_user_data_dir: Path to user data directory in container
+        """
+        self.mode = mode
+        self.image = image  # If None, defaults will be used from DockerUtils
+        self.registry_file = registry_file
+        self.persistent = persistent
+        self.remove_on_exit = remove_on_exit
+        self.network = network
+        self.volumes = volumes or []
+        self.env_vars = env_vars or {}
+        self.extra_args = extra_args or []
+        self.host_port = host_port
+        self.user_data_dir = user_data_dir
+        self.container_user_data_dir = container_user_data_dir
+    
+    def to_dict(self) -> Dict:
+        """Convert this configuration to a dictionary.
+        
+        Returns:
+            Dictionary representation of this configuration
+        """
+        return {
+            "mode": self.mode,
+            "image": self.image,
+            "registry_file": self.registry_file,
+            "persistent": self.persistent,
+            "remove_on_exit": self.remove_on_exit,
+            "network": self.network,
+            "volumes": self.volumes,
+            "env_vars": self.env_vars,
+            "extra_args": self.extra_args,
+            "host_port": self.host_port,
+            "user_data_dir": self.user_data_dir,
+            "container_user_data_dir": self.container_user_data_dir
+        }
+        
+    @staticmethod
+    def from_kwargs(kwargs: Dict) -> "DockerConfig":
+        """Create a DockerConfig from a dictionary of keyword arguments.
+        
+        Args:
+            kwargs: Dictionary of configuration options
+            
+        Returns:
+            New DockerConfig instance
+        """
+        return DockerConfig(
+            mode=kwargs.get("mode", "connect"),
+            image=kwargs.get("image"),
+            registry_file=kwargs.get("registry_file"),
+            persistent=kwargs.get("persistent", False),
+            remove_on_exit=kwargs.get("remove_on_exit", True),
+            network=kwargs.get("network"),
+            volumes=kwargs.get("volumes"),
+            env_vars=kwargs.get("env_vars"),
+            extra_args=kwargs.get("extra_args"),
+            host_port=kwargs.get("host_port"),
+            user_data_dir=kwargs.get("user_data_dir"),
+            container_user_data_dir=kwargs.get("container_user_data_dir", "/data")
+        )
+        
+    def clone(self, **kwargs) -> "DockerConfig":
+        """Create a copy of this configuration with updated values.
+        
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+            
+        Returns:
+            DockerConfig: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return DockerConfig.from_kwargs(config_dict)

crawl4ai/browser/docker_registry.py

@@ -0,0 +1,174 @@
+"""Docker registry module for Crawl4AI.
+
+This module provides a registry system for tracking and reusing Docker containers
+across browser sessions, improving performance and resource utilization.
+"""
+
+import os
+import json
+import time
+from typing import Dict, Optional
+
+from ..utils import get_home_folder
+
+
+class DockerRegistry:
+    """Manages a registry of Docker containers used for browser automation.
+    
+    This registry tracks containers by configuration hash, allowing reuse of appropriately
+    configured containers instead of creating new ones for each session.
+    
+    Attributes:
+        registry_file (str): Path to the registry file
+        containers (dict): Dictionary of container information
+        port_map (dict): Map of host ports to container IDs
+        last_port (int): Last port assigned
+    """
+    
+    def __init__(self, registry_file: Optional[str] = None):
+        """Initialize the registry with an optional path to the registry file.
+        
+        Args:
+            registry_file: Path to the registry file. If None, uses default path.
+        """
+        self.registry_file = registry_file or os.path.join(get_home_folder(), "docker_browser_registry.json")
+        self.containers = {}
+        self.port_map = {}
+        self.last_port = 9222
+        self.load()
+    
+    def load(self):
+        """Load container registry from file."""
+        if os.path.exists(self.registry_file):
+            try:
+                with open(self.registry_file, 'r') as f:
+                    registry_data = json.load(f)
+                    self.containers = registry_data.get("containers", {})
+                    self.port_map = registry_data.get("ports", {})
+                    self.last_port = registry_data.get("last_port", 9222)
+            except Exception:
+                # Reset to defaults on error
+                self.containers = {}
+                self.port_map = {}
+                self.last_port = 9222
+        else:
+            # Initialize with defaults if file doesn't exist
+            self.containers = {}
+            self.port_map = {}
+            self.last_port = 9222
+    
+    def save(self):
+        """Save container registry to file."""
+        os.makedirs(os.path.dirname(self.registry_file), exist_ok=True)
+        with open(self.registry_file, 'w') as f:
+            json.dump({
+                "containers": self.containers,
+                "ports": self.port_map,
+                "last_port": self.last_port
+            }, f, indent=2)
+    
+    def register_container(self, container_id: str, host_port: int, config_hash: str):
+        """Register a container with its configuration hash and port mapping.
+        
+        Args:
+            container_id: Docker container ID
+            host_port: Host port mapped to container
+            config_hash: Hash of configuration used to create container
+        """
+        self.containers[container_id] = {
+            "host_port": host_port,
+            "config_hash": config_hash,
+            "created_at": time.time()
+        }
+        self.port_map[str(host_port)] = container_id
+        self.save()
+    
+    def unregister_container(self, container_id: str):
+        """Unregister a container.
+        
+        Args:
+            container_id: Docker container ID to unregister
+        """
+        if container_id in self.containers:
+            host_port = self.containers[container_id]["host_port"]
+            if str(host_port) in self.port_map:
+                del self.port_map[str(host_port)]
+            del self.containers[container_id]
+            self.save()
+    
+    def find_container_by_config(self, config_hash: str, docker_utils) -> Optional[str]:
+        """Find a container that matches the given configuration hash.
+        
+        Args:
+            config_hash: Hash of configuration to match
+            docker_utils: DockerUtils instance to check running containers
+            
+        Returns:
+            Container ID if found, None otherwise
+        """
+        for container_id, data in self.containers.items():
+            if data["config_hash"] == config_hash and docker_utils.is_container_running(container_id):
+                return container_id
+        return None
+    
+    def get_container_host_port(self, container_id: str) -> Optional[int]:
+        """Get the host port mapped to the container.
+        
+        Args:
+            container_id: Docker container ID
+            
+        Returns:
+            Host port if container is registered, None otherwise
+        """
+        if container_id in self.containers:
+            return self.containers[container_id]["host_port"]
+        return None
+    
+    def get_next_available_port(self, docker_utils) -> int:
+        """Get the next available host port for Docker mapping.
+        
+        Args:
+            docker_utils: DockerUtils instance to check port availability
+            
+        Returns:
+            Available port number
+        """
+        # Start from last port + 1
+        port = self.last_port + 1
+        
+        # Check if port is in use (either in our registry or system-wide)
+        while port in self.port_map or docker_utils.is_port_in_use(port):
+            port += 1
+        
+        # Update last port
+        self.last_port = port
+        self.save()
+        
+        return port
+    
+    def get_container_config_hash(self, container_id: str) -> Optional[str]:
+        """Get the configuration hash for a container.
+        
+        Args:
+            container_id: Docker container ID
+            
+        Returns:
+            Configuration hash if container is registered, None otherwise
+        """
+        if container_id in self.containers:
+            return self.containers[container_id]["config_hash"]
+        return None
+    
+    def cleanup_stale_containers(self, docker_utils):
+        """Clean up containers that are no longer running.
+        
+        Args:
+            docker_utils: DockerUtils instance to check container status
+        """
+        to_remove = []
+        for container_id in self.containers:
+            if not docker_utils.is_container_running(container_id):
+                to_remove.append(container_id)
+                
+        for container_id in to_remove:
+            self.unregister_container(container_id)

crawl4ai/browser/docker_strategy.py

@@ -0,0 +1,286 @@
+"""Docker browser strategy module for Crawl4AI.
+
+This module provides browser strategies for running browsers in Docker containers,
+which offers better isolation, consistency across platforms, and easy scaling.
+"""
+
+import os
+import uuid
+import asyncio
+from typing import Dict, List, Optional, Tuple, Union
+from pathlib import Path
+
+from playwright.async_api import Page, BrowserContext
+
+from ..async_logger import AsyncLogger
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+from .docker_config import DockerConfig
+from .docker_registry import DockerRegistry
+from .docker_utils import DockerUtils
+from .strategies import BuiltinBrowserStrategy
+
+
+class DockerBrowserStrategy(BuiltinBrowserStrategy):
+    """Docker-based browser strategy.
+    
+    Extends the BuiltinBrowserStrategy to run browsers in Docker containers.
+    Supports two modes:
+    1. "connect" - Uses a Docker image with Chrome already running
+    2. "launch" - Starts Chrome within the container with custom settings
+    
+    Attributes:
+        docker_config: Docker-specific configuration options
+        container_id: ID of current Docker container
+        container_name: Name assigned to the container
+        registry: Registry for tracking and reusing containers
+        docker_utils: Utilities for Docker operations
+        chrome_process_id: Process ID of Chrome within container
+        socat_process_id: Process ID of socat within container
+        internal_cdp_port: Chrome's internal CDP port
+        internal_mapped_port: Port that socat maps to internally
+    """
+    
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the Docker browser strategy.
+        
+        Args:
+            config: Browser configuration including Docker-specific settings
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+        
+        # Initialize Docker-specific attributes
+        self.docker_config = self.config.docker_config or DockerConfig()
+        self.container_id = None
+        self.container_name = f"crawl4ai-browser-{uuid.uuid4().hex[:8]}"
+        self.registry = DockerRegistry(self.docker_config.registry_file)
+        self.docker_utils = DockerUtils(logger)
+        self.chrome_process_id = None
+        self.socat_process_id = None
+        self.internal_cdp_port = 9222  # Chrome's internal CDP port
+        self.internal_mapped_port = 9223  # Port that socat maps to internally
+        self.shutting_down = False
+    
+    async def _generate_config_hash(self) -> str:
+        """Generate a hash of the configuration for container matching.
+        
+        Returns:
+            Hash string uniquely identifying this configuration
+        """
+        # Create a dict with the relevant parts of the config
+        config_dict = {
+            "image": self.docker_config.image,
+            "mode": self.docker_config.mode,
+            "browser_type": self.config.browser_type,
+            "headless": self.config.headless,
+        }
+        
+        # Add browser-specific config if in launch mode
+        if self.docker_config.mode == "launch":
+            config_dict.update({
+                "text_mode": self.config.text_mode,
+                "light_mode": self.config.light_mode,
+                "viewport_width": self.config.viewport_width,
+                "viewport_height": self.config.viewport_height,
+            })
+        
+        # Use the utility method to generate the hash
+        return self.docker_utils.generate_config_hash(config_dict)
+    
+    async def _get_or_create_cdp_url(self) -> str:
+        """Get CDP URL by either creating a new container or using an existing one.
+        
+        Returns:
+            CDP URL for connecting to the browser
+            
+        Raises:
+            Exception: If container creation or browser launch fails
+        """
+        # If CDP URL is explicitly provided, use it
+        if self.config.cdp_url:
+            return self.config.cdp_url
+        
+        # Ensure Docker image exists (will build if needed)
+        image_name = await self.docker_utils.ensure_docker_image_exists(
+            self.docker_config.image, 
+            self.docker_config.mode
+        )
+        
+        # Generate config hash for container matching
+        config_hash = await self._generate_config_hash()
+        
+        # Look for existing container with matching config
+        container_id = self.registry.find_container_by_config(config_hash, self.docker_utils)
+        
+        if container_id:
+            # Use existing container
+            self.container_id = container_id
+            host_port = self.registry.get_container_host_port(container_id)
+            if self.logger:
+                self.logger.info(f"Using existing Docker container: {container_id[:12]}", tag="DOCKER")
+        else:
+            # Get a port for the new container
+            host_port = self.docker_config.host_port or self.registry.get_next_available_port(self.docker_utils)
+            
+            # Prepare volumes list
+            volumes = list(self.docker_config.volumes)
+            
+            # Add user data directory if specified
+            if self.docker_config.user_data_dir:
+                # Ensure user data directory exists
+                os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
+                volumes.append(f"{self.docker_config.user_data_dir}:{self.docker_config.container_user_data_dir}")
+                
+                # Update config user_data_dir to point to container path
+                self.config.user_data_dir = self.docker_config.container_user_data_dir
+            
+            # Create a new container
+            container_id = await self.docker_utils.create_container(
+                image_name=image_name,
+                host_port=host_port,
+                container_name=self.container_name,
+                volumes=volumes,
+                network=self.docker_config.network,
+                env_vars=self.docker_config.env_vars,
+                extra_args=self.docker_config.extra_args
+            )
+            
+            if not container_id:
+                raise Exception("Failed to create Docker container")
+            
+            self.container_id = container_id
+            
+            # Register the container
+            self.registry.register_container(container_id, host_port, config_hash)
+            
+            # Wait for container to be ready
+            await self.docker_utils.wait_for_container_ready(container_id)
+            
+            # Handle specific setup based on mode
+            if self.docker_config.mode == "launch":
+                # In launch mode, we need to start socat and Chrome
+                await self.docker_utils.start_socat_in_container(container_id)
+                
+                # Build browser arguments
+                browser_args = self._build_browser_args()
+                
+                # Launch Chrome
+                await self.docker_utils.launch_chrome_in_container(container_id, browser_args)
+                
+                # Get PIDs for later cleanup
+                self.chrome_process_id = await self.docker_utils.get_process_id_in_container(
+                    container_id, "chrome"
+                )
+                self.socat_process_id = await self.docker_utils.get_process_id_in_container(
+                    container_id, "socat"
+                )
+            
+            # Wait for CDP to be ready
+            await self.docker_utils.wait_for_cdp_ready(host_port)
+            
+            if self.logger:
+                self.logger.success(f"Docker container ready: {container_id[:12]} on port {host_port}", tag="DOCKER")
+        
+        # Return CDP URL
+        return f"http://localhost:{host_port}"
+    
+    def _build_browser_args(self) -> List[str]:
+        """Build Chrome command line arguments based on BrowserConfig.
+        
+        Returns:
+            List of command line arguments for Chrome
+        """
+        args = [
+            "--no-sandbox",
+            "--disable-gpu",
+            f"--remote-debugging-port={self.internal_cdp_port}",
+            "--remote-debugging-address=0.0.0.0",  # Allow external connections
+            "--disable-dev-shm-usage",
+        ]
+        
+        if self.config.headless:
+            args.append("--headless=new")
+            
+        if self.config.viewport_width and self.config.viewport_height:
+            args.append(f"--window-size={self.config.viewport_width},{self.config.viewport_height}")
+            
+        if self.config.user_agent:
+            args.append(f"--user-agent={self.config.user_agent}")
+            
+        if self.config.text_mode:
+            args.extend([
+                "--blink-settings=imagesEnabled=false",
+                "--disable-remote-fonts",
+                "--disable-images",
+                "--disable-javascript",
+            ])
+            
+        if self.config.light_mode:
+            # Import here to avoid circular import
+            from .utils import get_browser_disable_options
+            args.extend(get_browser_disable_options())
+            
+        if self.config.user_data_dir:
+            args.append(f"--user-data-dir={self.config.user_data_dir}")
+            
+        if self.config.extra_args:
+            args.extend(self.config.extra_args)
+            
+        return args
+    
+    async def close(self):
+        """Close the browser and clean up Docker container if needed."""
+        # Set shutting_down flag to prevent race conditions
+        self.shutting_down = True
+        
+        # Store state if needed before closing
+        if self.browser and self.docker_config.user_data_dir and self.docker_config.persistent:
+            for context in self.browser.contexts:
+                try:
+                    storage_path = os.path.join(self.docker_config.user_data_dir, "storage_state.json")
+                    await context.storage_state(path=storage_path)
+                    if self.logger:
+                        self.logger.debug("Persisted storage state before closing browser", tag="DOCKER")
+                except Exception as e:
+                    if self.logger:
+                        self.logger.warning(
+                            message="Failed to persist storage state: {error}",
+                            tag="DOCKER",
+                            params={"error": str(e)}
+                        )
+        
+        # Close browser connection (but not container)
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+        
+        # Only clean up container if not persistent
+        if self.container_id and not self.docker_config.persistent:
+            # Stop Chrome process in "launch" mode
+            if self.docker_config.mode == "launch" and self.chrome_process_id:
+                await self.docker_utils.stop_process_in_container(
+                    self.container_id, self.chrome_process_id
+                )
+            
+            # Stop socat process in "launch" mode
+            if self.docker_config.mode == "launch" and self.socat_process_id:
+                await self.docker_utils.stop_process_in_container(
+                    self.container_id, self.socat_process_id
+                )
+            
+            # Remove or stop container based on configuration
+            if self.docker_config.remove_on_exit:
+                await self.docker_utils.remove_container(self.container_id)
+                # Unregister from registry
+                self.registry.unregister_container(self.container_id)
+            else:
+                await self.docker_utils.stop_container(self.container_id)
+            
+            self.container_id = None
+        
+        # Close Playwright
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None
+        
+        self.shutting_down = False

crawl4ai/browser/docker_utils.py

@@ -0,0 +1,582 @@
+import os
+import json
+import asyncio
+import hashlib
+import tempfile
+import shutil
+import socket
+import subprocess
+from typing import Dict, List, Optional, Tuple, Union
+
+class DockerUtils:
+    """Utility class for Docker operations in browser automation.
+    
+    This class provides methods for managing Docker images, containers,
+    and related operations needed for browser automation. It handles
+    image building, container lifecycle, port management, and registry operations.
+    
+    Attributes:
+        DOCKER_FOLDER (str): Path to folder containing Docker files
+        DOCKER_CONNECT_FILE (str): Path to Dockerfile for connect mode
+        DOCKER_LAUNCH_FILE (str): Path to Dockerfile for launch mode
+        DOCKER_START_SCRIPT (str): Path to startup script for connect mode
+        DEFAULT_CONNECT_IMAGE (str): Default image name for connect mode
+        DEFAULT_LAUNCH_IMAGE (str): Default image name for launch mode
+        logger: Optional logger instance
+    """
+    
+    # File paths for Docker resources
+    DOCKER_FOLDER = os.path.join(os.path.dirname(__file__), "docker")
+    DOCKER_CONNECT_FILE = os.path.join(DOCKER_FOLDER, "connect.Dockerfile")
+    DOCKER_LAUNCH_FILE = os.path.join(DOCKER_FOLDER, "launch.Dockerfile")
+    DOCKER_START_SCRIPT = os.path.join(DOCKER_FOLDER, "start.sh")
+    
+    # Default image names
+    DEFAULT_CONNECT_IMAGE = "crawl4ai/browser-connect:latest"
+    DEFAULT_LAUNCH_IMAGE = "crawl4ai/browser-launch:latest"
+    
+    def __init__(self, logger=None):
+        """Initialize Docker utilities.
+        
+        Args:
+            logger: Optional logger for recording operations
+        """
+        self.logger = logger
+    
+    # Image Management Methods
+    
+    async def check_image_exists(self, image_name: str) -> bool:
+        """Check if a Docker image exists.
+        
+        Args:
+            image_name: Name of the Docker image to check
+            
+        Returns:
+            bool: True if the image exists, False otherwise
+        """
+        cmd = ["docker", "image", "inspect", image_name]
+        
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            _, _ = await process.communicate()
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.debug(f"Error checking if image exists: {str(e)}", tag="DOCKER")
+            return False
+    
+    async def build_docker_image(self, image_name: str, dockerfile_path: str, 
+                              files_to_copy: Dict[str, str] = None) -> bool:
+        """Build a Docker image from a Dockerfile.
+        
+        Args:
+            image_name: Name to give the built image
+            dockerfile_path: Path to the Dockerfile
+            files_to_copy: Dict of {dest_name: source_path} for files to copy to build context
+            
+        Returns:
+            bool: True if image was built successfully, False otherwise
+        """
+        # Create a temporary build context
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Copy the Dockerfile
+            shutil.copy(dockerfile_path, os.path.join(temp_dir, "Dockerfile"))
+            
+            # Copy any additional files needed
+            if files_to_copy:
+                for dest_name, source_path in files_to_copy.items():
+                    shutil.copy(source_path, os.path.join(temp_dir, dest_name))
+            
+            # Build the image
+            cmd = [
+                "docker", "build",
+                "-t", image_name,
+                temp_dir
+            ]
+            
+            if self.logger:
+                self.logger.debug(f"Building Docker image with command: {' '.join(cmd)}", tag="DOCKER")
+            
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+            
+            if process.returncode != 0:
+                if self.logger:
+                    self.logger.error(
+                        message="Failed to build Docker image: {error}",
+                        tag="DOCKER",
+                        params={"error": stderr.decode()}
+                    )
+                return False
+            
+            if self.logger:
+                self.logger.success(f"Successfully built Docker image: {image_name}", tag="DOCKER")
+            return True
+    
+    async def ensure_docker_image_exists(self, image_name: str, mode: str = "connect") -> str:
+        """Ensure the required Docker image exists, creating it if necessary.
+        
+        Args:
+            image_name: Name of the Docker image
+            mode: Either "connect" or "launch" to determine which image to build
+            
+        Returns:
+            str: Name of the available Docker image
+            
+        Raises:
+            Exception: If image doesn't exist and can't be built
+        """
+        # If image name is not specified, use default based on mode
+        if not image_name:
+            image_name = self.DEFAULT_CONNECT_IMAGE if mode == "connect" else self.DEFAULT_LAUNCH_IMAGE
+        
+        # Check if the image already exists
+        if await self.check_image_exists(image_name):
+            if self.logger:
+                self.logger.debug(f"Docker image {image_name} already exists", tag="DOCKER")
+            return image_name
+        
+        # If we're using a custom image that doesn't exist, warn and fail
+        if (image_name != self.DEFAULT_CONNECT_IMAGE and image_name != self.DEFAULT_LAUNCH_IMAGE):
+            if self.logger:
+                self.logger.warning(
+                    f"Custom Docker image {image_name} not found and cannot be automatically created",
+                    tag="DOCKER"
+                )
+            raise Exception(f"Docker image {image_name} not found")
+        
+        # Build the appropriate default image
+        if self.logger:
+            self.logger.info(f"Docker image {image_name} not found, creating it now...", tag="DOCKER")
+        
+        if mode == "connect":
+            success = await self.build_docker_image(
+                image_name, 
+                self.DOCKER_CONNECT_FILE, 
+                {"start.sh": self.DOCKER_START_SCRIPT}
+            )
+        else:
+            success = await self.build_docker_image(
+                image_name, 
+                self.DOCKER_LAUNCH_FILE
+            )
+        
+        if not success:
+            raise Exception(f"Failed to create Docker image {image_name}")
+        
+        return image_name
+    
+    # Container Management Methods
+    
+    async def create_container(self, image_name: str, host_port: int, 
+                            container_name: Optional[str] = None,
+                            volumes: List[str] = None,
+                            network: Optional[str] = None,
+                            env_vars: Dict[str, str] = None,
+                            extra_args: List[str] = None) -> Optional[str]:
+        """Create a new Docker container.
+        
+        Args:
+            image_name: Docker image to use
+            host_port: Port on host to map to container port 9223
+            container_name: Optional name for the container
+            volumes: List of volume mappings (e.g., ["host_path:container_path"])
+            network: Optional Docker network to use
+            env_vars: Dictionary of environment variables
+            extra_args: Additional docker run arguments
+            
+        Returns:
+            str: Container ID if successful, None otherwise
+        """
+        # Prepare container command
+        cmd = [
+            "docker", "run",
+            "--detach",
+        ]
+        
+        # Add container name if specified
+        if container_name:
+            cmd.extend(["--name", container_name])
+        
+        # Add port mapping
+        cmd.extend(["-p", f"{host_port}:9223"])
+        
+        # Add volumes
+        if volumes:
+            for volume in volumes:
+                cmd.extend(["-v", volume])
+        
+        # Add network if specified
+        if network:
+            cmd.extend(["--network", network])
+        
+        # Add environment variables
+        if env_vars:
+            for key, value in env_vars.items():
+                cmd.extend(["-e", f"{key}={value}"])
+        
+        # Add extra args
+        if extra_args:
+            cmd.extend(extra_args)
+        
+        # Add image
+        cmd.append(image_name)
+        
+        if self.logger:
+            self.logger.debug(f"Creating Docker container with command: {' '.join(cmd)}", tag="DOCKER")
+        
+        # Run docker command
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+            
+            if process.returncode != 0:
+                if self.logger:
+                    self.logger.error(
+                        message="Failed to create Docker container: {error}",
+                        tag="DOCKER",
+                        params={"error": stderr.decode()}
+                    )
+                return None
+            
+            # Get container ID
+            container_id = stdout.decode().strip()
+            
+            if self.logger:
+                self.logger.success(f"Created Docker container: {container_id[:12]}", tag="DOCKER")
+            
+            return container_id
+            
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    message="Error creating Docker container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)}
+                )
+            return None
+    
+    async def is_container_running(self, container_id: str) -> bool:
+        """Check if a container is running.
+        
+        Args:
+            container_id: ID of the container to check
+            
+        Returns:
+            bool: True if the container is running, False otherwise
+        """
+        cmd = ["docker", "inspect", "--format", "{{.State.Running}}", container_id]
+        
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, _ = await process.communicate()
+            
+            return process.returncode == 0 and stdout.decode().strip() == "true"
+        except Exception as e:
+            if self.logger:
+                self.logger.debug(f"Error checking if container is running: {str(e)}", tag="DOCKER")
+            return False
+    
+    async def wait_for_container_ready(self, container_id: str, timeout: int = 30) -> bool:
+        """Wait for the container to be in running state.
+        
+        Args:
+            container_id: ID of the container to wait for
+            timeout: Maximum time to wait in seconds
+            
+        Returns:
+            bool: True if container is ready, False if timeout occurred
+        """
+        for _ in range(timeout):
+            if await self.is_container_running(container_id):
+                return True
+            await asyncio.sleep(1)
+        
+        if self.logger:
+            self.logger.warning(f"Container {container_id[:12]} not ready after {timeout}s timeout", tag="DOCKER")
+        return False
+    
+    async def stop_container(self, container_id: str) -> bool:
+        """Stop a Docker container.
+        
+        Args:
+            container_id: ID of the container to stop
+            
+        Returns:
+            bool: True if stopped successfully, False otherwise
+        """
+        cmd = ["docker", "stop", container_id]
+        
+        try:
+            process = await asyncio.create_subprocess_exec(*cmd)
+            await process.communicate()
+            
+            if self.logger:
+                self.logger.debug(f"Stopped container: {container_id[:12]}", tag="DOCKER")
+                
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to stop container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)}
+                )
+            return False
+    
+    async def remove_container(self, container_id: str, force: bool = True) -> bool:
+        """Remove a Docker container.
+        
+        Args:
+            container_id: ID of the container to remove
+            force: Whether to force removal
+            
+        Returns:
+            bool: True if removed successfully, False otherwise
+        """
+        cmd = ["docker", "rm"]
+        if force:
+            cmd.append("-f")
+        cmd.append(container_id)
+        
+        try:
+            process = await asyncio.create_subprocess_exec(*cmd)
+            await process.communicate()
+            
+            if self.logger:
+                self.logger.debug(f"Removed container: {container_id[:12]}", tag="DOCKER")
+                
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to remove container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)}
+                )
+            return False
+    
+    # Container Command Execution Methods
+    
+    async def exec_in_container(self, container_id: str, command: List[str], 
+                             detach: bool = False) -> Tuple[int, str, str]:
+        """Execute a command in a running container.
+        
+        Args:
+            container_id: ID of the container
+            command: Command to execute as a list of strings
+            detach: Whether to run the command in detached mode
+            
+        Returns:
+            Tuple of (return_code, stdout, stderr)
+        """
+        cmd = ["docker", "exec"]
+        if detach:
+            cmd.append("-d")
+        cmd.append(container_id)
+        cmd.extend(command)
+        
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+            
+            return process.returncode, stdout.decode(), stderr.decode()
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    message="Error executing command in container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)}
+                )
+            return -1, "", str(e)
+    
+    async def start_socat_in_container(self, container_id: str) -> bool:
+        """Start socat in the container to map port 9222 to 9223.
+        
+        Args:
+            container_id: ID of the container
+            
+        Returns:
+            bool: True if socat started successfully, False otherwise
+        """
+        # Command to run socat as a background process
+        cmd = ["socat", "TCP-LISTEN:9223,fork", "TCP:localhost:9222"]
+        
+        returncode, _, stderr = await self.exec_in_container(container_id, cmd, detach=True)
+        
+        if returncode != 0:
+            if self.logger:
+                self.logger.error(
+                    message="Failed to start socat in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr}
+                )
+            return False
+            
+        if self.logger:
+            self.logger.debug(f"Started socat in container: {container_id[:12]}", tag="DOCKER")
+        
+        # Wait a moment for socat to start
+        await asyncio.sleep(1)
+        return True
+    
+    async def launch_chrome_in_container(self, container_id: str, browser_args: List[str]) -> bool:
+        """Launch Chrome inside the container with specified arguments.
+        
+        Args:
+            container_id: ID of the container
+            browser_args: Chrome command line arguments
+            
+        Returns:
+            bool: True if Chrome started successfully, False otherwise
+        """
+        # Build Chrome command
+        chrome_cmd = ["google-chrome"]
+        chrome_cmd.extend(browser_args)
+        
+        returncode, _, stderr = await self.exec_in_container(container_id, chrome_cmd, detach=True)
+        
+        if returncode != 0:
+            if self.logger:
+                self.logger.error(
+                    message="Failed to launch Chrome in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr}
+                )
+            return False
+            
+        if self.logger:
+            self.logger.debug(f"Launched Chrome in container: {container_id[:12]}", tag="DOCKER")
+        
+        return True
+    
+    async def get_process_id_in_container(self, container_id: str, process_name: str) -> Optional[int]:
+        """Get the process ID for a process in the container.
+        
+        Args:
+            container_id: ID of the container
+            process_name: Name pattern to search for
+            
+        Returns:
+            int: Process ID if found, None otherwise
+        """
+        cmd = ["pgrep", "-f", process_name]
+        
+        returncode, stdout, _ = await self.exec_in_container(container_id, cmd)
+        
+        if returncode == 0 and stdout.strip():
+            pid = int(stdout.strip().split("\n")[0])
+            return pid
+        
+        return None
+    
+    async def stop_process_in_container(self, container_id: str, pid: int) -> bool:
+        """Stop a process in the container by PID.
+        
+        Args:
+            container_id: ID of the container
+            pid: Process ID to stop
+            
+        Returns:
+            bool: True if process was stopped, False otherwise
+        """
+        cmd = ["kill", "-TERM", str(pid)]
+        
+        returncode, _, stderr = await self.exec_in_container(container_id, cmd)
+        
+        if returncode != 0:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to stop process in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr}
+                )
+            return False
+            
+        if self.logger:
+            self.logger.debug(f"Stopped process {pid} in container: {container_id[:12]}", tag="DOCKER")
+        
+        return True
+    
+    # Network and Port Methods
+    
+    async def wait_for_cdp_ready(self, host_port: int, timeout: int = 30) -> bool:
+        """Wait for the CDP endpoint to be ready.
+        
+        Args:
+            host_port: Port to check for CDP endpoint
+            timeout: Maximum time to wait in seconds
+            
+        Returns:
+            bool: True if CDP endpoint is ready, False if timeout occurred
+        """
+        import aiohttp
+        
+        url = f"http://localhost:{host_port}/json/version"
+        
+        for _ in range(timeout):
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(url, timeout=1) as response:
+                        if response.status == 200:
+                            if self.logger:
+                                self.logger.debug(f"CDP endpoint ready on port {host_port}", tag="DOCKER")
+                            return True
+            except Exception:
+                pass
+            await asyncio.sleep(1)
+        
+        if self.logger:
+            self.logger.warning(f"CDP endpoint not ready on port {host_port} after {timeout}s timeout", tag="DOCKER")
+        return False
+    
+    def is_port_in_use(self, port: int) -> bool:
+        """Check if a port is already in use on the host.
+        
+        Args:
+            port: Port number to check
+            
+        Returns:
+            bool: True if port is in use, False otherwise
+        """
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            return s.connect_ex(('localhost', port)) == 0
+    
+    def get_next_available_port(self, start_port: int = 9223) -> int:
+        """Get the next available port starting from a given port.
+        
+        Args:
+            start_port: Port number to start checking from
+            
+        Returns:
+            int: First available port number
+        """
+        port = start_port
+        while self.is_port_in_use(port):
+            port += 1
+        return port
+    
+    # Configuration Hash Methods
+    
+    def generate_config_hash(self, config_dict: Dict) -> str:
+        """Generate a hash of the configuration for container matching.
+        
+        Args:
+            config_dict: Dictionary of configuration parameters
+            
+        Returns:
+            str: Hash string uniquely identifying this configuration
+        """
+        # Convert to canonical JSON string and hash
+        config_json = json.dumps(config_dict, sort_keys=True)
+        return hashlib.sha256(config_json.encode()).hexdigest()

crawl4ai/browser/manager.py

@@ -0,0 +1,204 @@
+"""Browser manager module for Crawl4AI.
+
+This module provides a central browser management class that uses the
+strategy pattern internally while maintaining the existing API.
+It also implements a page pooling mechanism for improved performance.
+"""
+
+import asyncio
+import time
+from typing import Optional, Tuple, List
+
+from playwright.async_api import Page, BrowserContext
+
+from ..async_logger import AsyncLogger
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+
+from .strategies import (
+    BaseBrowserStrategy,
+    PlaywrightBrowserStrategy,
+    CDPBrowserStrategy,
+    BuiltinBrowserStrategy
+)
+
+# Import DockerBrowserStrategy if available
+try:
+    from .docker_strategy import DockerBrowserStrategy
+except ImportError:
+    DockerBrowserStrategy = None
+
+class BrowserManager:
+    """Main interface for browser management in Crawl4AI.
+    
+    This class maintains backward compatibility with the existing implementation
+    while using the strategy pattern internally for different browser types.
+    
+    Attributes:
+        config (BrowserConfig): Configuration object containing all browser settings
+        logger: Logger instance for recording events and errors
+        browser: The browser instance
+        default_context: The default browser context
+        managed_browser: The managed browser instance
+        playwright: The Playwright instance
+        sessions: Dictionary to store session information
+        session_ttl: Session timeout in seconds
+    """
+    
+    def __init__(self, browser_config: Optional[BrowserConfig] = None, logger: Optional[AsyncLogger] = None):
+        """Initialize the BrowserManager with a browser configuration.
+        
+        Args:
+            browser_config: Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+        """
+        self.config = browser_config or BrowserConfig()
+        self.logger = logger
+        
+        # Create strategy based on configuration
+        self._strategy = self._create_strategy()
+        
+        # Initialize state variables for compatibility with existing code
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        
+        # For session management (from existing implementation)
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+    
+    def _create_strategy(self) -> BaseBrowserStrategy:
+        """Create appropriate browser strategy based on configuration.
+        
+        Returns:
+            BaseBrowserStrategy: The selected browser strategy
+        """
+        if self.config.browser_mode == "builtin":
+            return BuiltinBrowserStrategy(self.config, self.logger)
+        elif self.config.browser_mode == "docker":
+            if DockerBrowserStrategy is None:
+                if self.logger:
+                    self.logger.error(
+                        "Docker browser strategy requested but not available. "
+                        "Falling back to PlaywrightBrowserStrategy.",
+                        tag="BROWSER"
+                    )
+                return PlaywrightBrowserStrategy(self.config, self.logger)
+            return DockerBrowserStrategy(self.config, self.logger)
+        elif self.config.cdp_url or self.config.use_managed_browser:
+            return CDPBrowserStrategy(self.config, self.logger)
+        else:
+            return PlaywrightBrowserStrategy(self.config, self.logger)
+    
+    async def start(self):
+        """Start the browser instance and set up the default context.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Start the strategy
+        await self._strategy.start()
+        
+        # Update legacy references
+        self.browser = self._strategy.browser
+        self.default_context = self._strategy.default_context
+        
+        # Set browser process reference (for CDP strategy)
+        if hasattr(self._strategy, 'browser_process'):
+            self.managed_browser = self._strategy
+        
+        # Set Playwright reference
+        self.playwright = self._strategy.playwright
+        
+        # Sync sessions if needed
+        if hasattr(self._strategy, 'sessions'):
+            self.sessions = self._strategy.sessions
+            self.session_ttl = self._strategy.session_ttl
+        
+        return self
+    
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page for the given configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Delegate to strategy
+        page, context = await self._strategy.get_page(crawlerRunConfig)
+        
+        # Sync sessions if needed
+        if hasattr(self._strategy, 'sessions'):
+            self.sessions = self._strategy.sessions
+        
+        return page, context
+        
+    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
+        """Get multiple pages with the same configuration.
+        
+        This method efficiently creates multiple browser pages using the same configuration,
+        which is useful for parallel crawling of multiple URLs.
+        
+        Args:
+            crawlerRunConfig: Configuration for the pages
+            count: Number of pages to create
+            
+        Returns:
+            List of (Page, Context) tuples
+        """
+        # Delegate to strategy
+        pages = await self._strategy.get_pages(crawlerRunConfig, count)
+        
+        # Sync sessions if needed
+        if hasattr(self._strategy, 'sessions'):
+            self.sessions = self._strategy.sessions
+            
+        return pages
+    
+    async def kill_session(self, session_id: str):
+        """Kill a browser session and clean up resources.
+        
+        Args:
+            session_id: The session ID to kill
+        """
+        # Handle kill_session via our strategy if it supports it
+        if hasattr(self._strategy, '_kill_session'):
+            await self._strategy._kill_session(session_id)
+        elif session_id in self.sessions:
+            context, page, _ = self.sessions[session_id]
+            await page.close()
+            # Only close context if not using CDP
+            if not self.config.use_managed_browser and not self.config.cdp_url and not self.config.browser_mode == "builtin":
+                await context.close()
+            del self.sessions[session_id]
+    
+    def _cleanup_expired_sessions(self):
+        """Clean up expired sessions based on TTL."""
+        # Use strategy's implementation if available
+        if hasattr(self._strategy, '_cleanup_expired_sessions'):
+            self._strategy._cleanup_expired_sessions()
+            return
+        
+        # Otherwise use our own implementation
+        current_time = time.time()
+        expired_sessions = [
+            sid
+            for sid, (_, _, last_used) in self.sessions.items()
+            if current_time - last_used > self.session_ttl
+        ]
+        for sid in expired_sessions:
+            asyncio.create_task(self.kill_session(sid))
+    
+    async def close(self):
+        """Close the browser and clean up resources."""
+        # Delegate to strategy
+        await self._strategy.close()
+        
+        # Reset legacy references
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        self.sessions = {}

crawl4ai/browser/profiles.py

@@ -0,0 +1,457 @@
+"""Browser profile management module for Crawl4AI.
+
+This module provides functionality for creating and managing browser profiles
+that can be used for authenticated browsing.
+"""
+
+import os
+import asyncio
+import signal
+import sys
+import datetime
+import uuid
+import shutil
+from typing import List, Dict, Optional, Any
+from colorama import Fore, Style, init
+
+from ..async_configs import BrowserConfig
+from ..async_logger import AsyncLogger, AsyncLoggerBase
+from ..utils import get_home_folder
+
+class BrowserProfileManager:
+    """Manages browser profiles for Crawl4AI.
+    
+    This class provides functionality to create and manage browser profiles
+    that can be used for authenticated browsing with Crawl4AI.
+    
+    Profiles are stored by default in ~/.crawl4ai/profiles/
+    """
+    
+    def __init__(self, logger: Optional[AsyncLoggerBase] = None):
+        """Initialize the BrowserProfileManager.
+        
+        Args:
+            logger: Logger for outputting messages. If None, a default AsyncLogger is created.
+        """
+        # Initialize colorama for colorful terminal output
+        init()
+        
+        # Create a logger if not provided
+        if logger is None:
+            self.logger = AsyncLogger(verbose=True)
+        elif not isinstance(logger, AsyncLoggerBase):
+            self.logger = AsyncLogger(verbose=True)
+        else:
+            self.logger = logger
+            
+        # Ensure profiles directory exists
+        self.profiles_dir = os.path.join(get_home_folder(), "profiles")
+        os.makedirs(self.profiles_dir, exist_ok=True)
+    
+    async def create_profile(self, 
+                          profile_name: Optional[str] = None, 
+                          browser_config: Optional[BrowserConfig] = None) -> Optional[str]:
+        """Create a browser profile interactively.
+        
+        Args:
+            profile_name: Name for the profile. If None, a name is generated.
+            browser_config: Configuration for the browser. If None, a default configuration is used.
+                
+        Returns:
+            Path to the created profile directory, or None if creation failed
+        """
+        # Create default browser config if none provided
+        if browser_config is None:
+            browser_config = BrowserConfig(
+                browser_type="chromium",
+                headless=False,  # Must be visible for user interaction
+                verbose=True
+            )
+        else:
+            # Ensure headless is False for user interaction
+            browser_config.headless = False
+            
+        # Generate profile name if not provided
+        if not profile_name:
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"profile_{timestamp}_{uuid.uuid4().hex[:6]}"
+            
+        # Sanitize profile name (replace spaces and special chars)
+        profile_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in profile_name)
+        
+        # Set user data directory
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        os.makedirs(profile_path, exist_ok=True)
+        
+        # Print instructions for the user with colorama formatting
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="PROFILE")
+        self.logger.info(f"Creating browser profile: {Fore.GREEN}{profile_name}{Style.RESET_ALL}", tag="PROFILE")
+        self.logger.info(f"Profile directory: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+        
+        self.logger.info("\nInstructions:", tag="PROFILE")
+        self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE")
+        self.logger.info(f"2. {Fore.CYAN}Log in to websites{Style.RESET_ALL}, configure settings, etc. as needed.", tag="PROFILE")
+        self.logger.info(f"3. When you're done, {Fore.YELLOW}press 'q' in this terminal{Style.RESET_ALL} to close the browser.", tag="PROFILE")
+        self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE")
+        self.logger.info(f"{border}\n", tag="PROFILE")
+        
+        # Import the necessary classes with local imports to avoid circular references
+        from .strategies import CDPBrowserStrategy
+        
+        # Set browser config to use the profile path
+        browser_config.user_data_dir = profile_path
+        
+        # Create a CDP browser strategy for the profile creation
+        browser_strategy = CDPBrowserStrategy(browser_config, self.logger)
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="PROFILE")
+            await browser_strategy.close()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Profile creation interrupted. Profile may be incomplete.", tag="PROFILE")
+                sys.exit(1)
+                
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user is done with the browser
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' when you've finished using the browser...{Style.RESET_ALL}", tag="PROFILE")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser and saving profile...{Style.RESET_ALL}", tag="PROFILE")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if browser_strategy.browser_process and browser_strategy.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="PROFILE")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+        
+        try:
+            # Start the browser
+            await browser_strategy.start()
+            
+            # Check if browser started successfully
+            if not browser_strategy.browser_process:
+                self.logger.error("Failed to start browser process.", tag="PROFILE")
+                return None
+            
+            self.logger.info(f"Browser launched. {Fore.CYAN}Waiting for you to finish...{Style.RESET_ALL}", tag="PROFILE") 
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for either the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_strategy.browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_strategy.browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="PROFILE")
+                await browser_strategy.close()
+            
+            self.logger.success(f"Browser closed. Profile saved at: {Fore.GREEN}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+                
+        except Exception as e:
+            self.logger.error(f"Error creating profile: {str(e)}", tag="PROFILE")
+            await browser_strategy.close()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await browser_strategy.close()
+        
+        # Return the profile path
+        return profile_path
+    
+    def list_profiles(self) -> List[Dict[str, Any]]:
+        """List all available browser profiles.
+        
+        Returns:
+            List of dictionaries containing profile information
+        """
+        if not os.path.exists(self.profiles_dir):
+            return []
+            
+        profiles = []
+        
+        for name in os.listdir(self.profiles_dir):
+            profile_path = os.path.join(self.profiles_dir, name)
+            
+            # Skip if not a directory
+            if not os.path.isdir(profile_path):
+                continue
+                
+            # Check if this looks like a valid browser profile
+            # For Chromium: Look for Preferences file
+            # For Firefox: Look for prefs.js file
+            is_valid = False
+            
+            if os.path.exists(os.path.join(profile_path, "Preferences")) or \
+               os.path.exists(os.path.join(profile_path, "Default", "Preferences")):
+                is_valid = "chromium"
+            elif os.path.exists(os.path.join(profile_path, "prefs.js")):
+                is_valid = "firefox"
+                
+            if is_valid:
+                # Get creation time
+                created = datetime.datetime.fromtimestamp(
+                    os.path.getctime(profile_path)
+                )
+                
+                profiles.append({
+                    "name": name,
+                    "path": profile_path,
+                    "created": created,
+                    "type": is_valid
+                })
+                
+        # Sort by creation time, newest first
+        profiles.sort(key=lambda x: x["created"], reverse=True)
+        
+        return profiles
+    
+    def get_profile_path(self, profile_name: str) -> Optional[str]:
+        """Get the full path to a profile by name.
+        
+        Args:
+            profile_name: Name of the profile (not the full path)
+            
+        Returns:
+            Full path to the profile directory, or None if not found
+        """
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            # Check if profile_name itself is full path
+            if os.path.isabs(profile_name):
+                profile_path = profile_name
+            else:
+                return None
+        
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return None  # Not a valid browser profile
+            
+        return profile_path
+    
+    def delete_profile(self, profile_name_or_path: str) -> bool:
+        """Delete a browser profile by name or path.
+        
+        Args:
+            profile_name_or_path: Name of the profile or full path to profile directory
+            
+        Returns:
+            True if the profile was deleted successfully, False otherwise
+        """
+        # Determine if input is a name or a path
+        if os.path.isabs(profile_name_or_path):
+            # Full path provided
+            profile_path = profile_name_or_path
+        else:
+            # Just a name provided, construct path
+            profile_path = os.path.join(self.profiles_dir, profile_name_or_path)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            return False
+            
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return False  # Not a valid browser profile
+            
+        # Delete the profile directory
+        try:
+            shutil.rmtree(profile_path)
+            return True
+        except Exception:
+            return False
+            
+    async def interactive_manager(self, crawl_callback=None):
+        """Launch an interactive profile management console.
+        
+        Args:
+            crawl_callback: Function to call when selecting option to use 
+                a profile for crawling. It will be called with (profile_path, url).
+        """
+        while True:
+            self.logger.info(f"\n{Fore.CYAN}Profile Management Options:{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"1. {Fore.GREEN}Create a new profile{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"2. {Fore.YELLOW}List available profiles{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"3. {Fore.RED}Delete a profile{Style.RESET_ALL}", tag="MENU")
+            
+            # Only show crawl option if callback provided
+            if crawl_callback:
+                self.logger.info(f"4. {Fore.CYAN}Use a profile to crawl a website{Style.RESET_ALL}", tag="MENU")
+                self.logger.info(f"5. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "5"
+            else:
+                self.logger.info(f"4. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "4"
+            
+            choice = input(f"\n{Fore.CYAN}Enter your choice (1-{exit_option}): {Style.RESET_ALL}")
+            
+            if choice == "1":
+                # Create new profile
+                name = input(f"{Fore.GREEN}Enter a name for the new profile (or press Enter for auto-generated name): {Style.RESET_ALL}")
+                await self.create_profile(name or None)
+                
+            elif choice == "2":
+                # List profiles
+                profiles = self.list_profiles()
+                
+                if not profiles:
+                    self.logger.warning("  No profiles found. Create one first with option 1.", tag="PROFILES")
+                    continue
+                
+                # Print profile information with colorama formatting
+                self.logger.info("\nAvailable profiles:", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {Fore.CYAN}{profile['name']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Path: {Fore.YELLOW}{profile['path']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES")
+                    self.logger.info(f"    Browser type: {profile['type']}", tag="PROFILES")
+                    self.logger.info("", tag="PROFILES")  # Empty line for spacing
+                
+            elif choice == "3":
+                # Delete profile
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found to delete", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to delete
+                profile_idx = input(f"{Fore.RED}Enter the number of the profile to delete (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_name = profiles[idx]["name"]
+                        self.logger.info(f"Deleting profile: {Fore.YELLOW}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                        
+                        # Confirm deletion
+                        confirm = input(f"{Fore.RED}Are you sure you want to delete this profile? (y/n): {Style.RESET_ALL}")
+                        if confirm.lower() == 'y':
+                            success = self.delete_profile(profiles[idx]["path"])
+                            
+                            if success:
+                                self.logger.success(f"Profile {Fore.GREEN}{profile_name}{Style.RESET_ALL} deleted successfully", tag="PROFILES")
+                            else:
+                                self.logger.error(f"Failed to delete profile {Fore.RED}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == "4" and crawl_callback:
+                # Use profile to crawl a site
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found. Create one first.", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to use
+                profile_idx = input(f"{Fore.CYAN}Enter the number of the profile to use (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_path = profiles[idx]["path"]
+                        url = input(f"{Fore.CYAN}Enter the URL to crawl: {Style.RESET_ALL}")
+                        if url:
+                            # Call the provided crawl callback
+                            await crawl_callback(profile_path, url)
+                        else:
+                            self.logger.error("No URL provided", tag="CRAWL")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif (choice == "4" and not crawl_callback) or (choice == "5" and crawl_callback):
+                # Exit
+                self.logger.info("Exiting profile management", tag="MENU")
+                break
+                
+            else:
+                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")

crawl4ai/browser/utils.py

@@ -0,0 +1,328 @@
+"""Browser utilities module for Crawl4AI.
+
+This module provides utility functions for browser management,
+including process management, CDP connection utilities,
+and Playwright instance management.
+"""
+
+import asyncio
+import os
+import sys
+import time
+import tempfile
+import subprocess
+from typing import Optional
+
+from playwright.async_api import async_playwright
+
+from ..utils import get_chromium_path
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+
+from ..async_logger import AsyncLogger
+
+
+_playwright_instance = None
+
+async def get_playwright():
+    """Get or create the Playwright instance (singleton pattern).
+    
+    Returns:
+        Playwright: The Playwright instance
+    """
+    global _playwright_instance
+    if _playwright_instance is None or True:
+        _playwright_instance = await async_playwright().start()
+    return _playwright_instance
+
+async def get_browser_executable(browser_type: str) -> str:
+    """Get the path to browser executable, with platform-specific handling.
+    
+    Args:
+        browser_type: Type of browser (chromium, firefox, webkit)
+        
+    Returns:
+        Path to browser executable
+    """
+    return await get_chromium_path(browser_type)
+
+def create_temp_directory(prefix="browser-profile-") -> str:
+    """Create a temporary directory for browser data.
+    
+    Args:
+        prefix: Prefix for the temporary directory name
+        
+    Returns:
+        Path to the created temporary directory
+    """
+    return tempfile.mkdtemp(prefix=prefix)
+
+def is_windows() -> bool:
+    """Check if the current platform is Windows.
+    
+    Returns:
+        True if Windows, False otherwise
+    """
+    return sys.platform == "win32"
+
+def is_macos() -> bool:
+    """Check if the current platform is macOS.
+    
+    Returns:
+        True if macOS, False otherwise
+    """
+    return sys.platform == "darwin"
+
+def is_linux() -> bool:
+    """Check if the current platform is Linux.
+    
+    Returns:
+        True if Linux, False otherwise
+    """
+    return not (is_windows() or is_macos())
+    
+def is_browser_running(pid: Optional[int]) -> bool:
+    """Check if a process with the given PID is running.
+    
+    Args:
+        pid: Process ID to check
+        
+    Returns:
+        bool: True if the process is running, False otherwise
+    """
+    if not pid:
+        return False
+        
+    try:
+        # Check if the process exists
+        if is_windows():
+            process = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], 
+                                     capture_output=True, text=True)
+            return str(pid) in process.stdout
+        else:
+            # Unix-like systems
+            os.kill(pid, 0)  # This doesn't actually kill the process, just checks if it exists
+        return True
+    except (ProcessLookupError, PermissionError, OSError):
+        return False
+
+def get_browser_disable_options() -> list:
+    """Get standard list of browser disable options for performance.
+    
+    Returns:
+        List of command-line options to disable various browser features
+    """
+    return [
+        "--disable-background-networking",
+        "--disable-background-timer-throttling",
+        "--disable-backgrounding-occluded-windows",
+        "--disable-breakpad",
+        "--disable-client-side-phishing-detection",
+        "--disable-component-extensions-with-background-pages",
+        "--disable-default-apps",
+        "--disable-extensions",
+        "--disable-features=TranslateUI",
+        "--disable-hang-monitor",
+        "--disable-ipc-flooding-protection",
+        "--disable-popup-blocking",
+        "--disable-prompt-on-repost",
+        "--disable-sync",
+        "--force-color-profile=srgb",
+        "--metrics-recording-only",
+        "--no-first-run",
+        "--password-store=basic",
+        "--use-mock-keychain",
+    ]
+
+
+async def find_optimal_browser_config(total_urls=50, verbose=True, rate_limit_delay=0.2):
+    """Find optimal browser configuration for crawling a specific number of URLs.
+    
+    Args:
+        total_urls: Number of URLs to crawl
+        verbose: Whether to print progress
+        rate_limit_delay: Delay between page loads to avoid rate limiting
+        
+    Returns:
+        dict: Contains fastest, lowest_memory, and optimal configurations
+    """
+    from .manager import BrowserManager
+    if verbose:
+        print(f"\n=== Finding optimal configuration for crawling {total_urls} URLs ===\n")
+    
+    # Generate test URLs with timestamp to avoid caching
+    timestamp = int(time.time())
+    urls = [f"https://example.com/page_{i}?t={timestamp}" for i in range(total_urls)]
+    
+    # Limit browser configurations to test (1 browser to max 10)
+    max_browsers = min(10, total_urls)
+    configs_to_test = []
+    
+    # Generate configurations (browser count, pages distribution)
+    for num_browsers in range(1, max_browsers + 1):
+        base_pages = total_urls // num_browsers
+        remainder = total_urls % num_browsers
+        
+        # Create distribution array like [3, 3, 2, 2] (some browsers get one more page)
+        if remainder > 0:
+            distribution = [base_pages + 1] * remainder + [base_pages] * (num_browsers - remainder)
+        else:
+            distribution = [base_pages] * num_browsers
+            
+        configs_to_test.append((num_browsers, distribution))
+    
+    results = []
+    
+    # Test each configuration
+    for browser_count, page_distribution in configs_to_test:
+        if verbose:
+            print(f"Testing {browser_count} browsers with distribution {tuple(page_distribution)}")
+        
+        try:
+            # Track memory if possible
+            try:
+                import psutil
+                process = psutil.Process()
+                start_memory = process.memory_info().rss / (1024 * 1024)  # MB
+            except ImportError:
+                if verbose: 
+                    print("Memory tracking not available (psutil not installed)")
+                start_memory = 0
+            
+            # Start browsers in parallel
+            managers = []
+            start_tasks = []
+            start_time = time.time()
+
+            logger = AsyncLogger(verbose=True, log_file=None)
+            
+            for i in range(browser_count):
+                config = BrowserConfig(headless=True)
+                manager = BrowserManager(browser_config=config, logger=logger)
+                start_tasks.append(manager.start())
+                managers.append(manager)
+            
+            await asyncio.gather(*start_tasks)
+            
+            # Distribute URLs among browsers
+            urls_per_manager = {}
+            url_index = 0
+            
+            for i, manager in enumerate(managers):
+                pages_for_this_browser = page_distribution[i]
+                end_index = url_index + pages_for_this_browser
+                urls_per_manager[manager] = urls[url_index:end_index]
+                url_index = end_index
+            
+            # Create pages for each browser
+            all_pages = []
+            for manager, manager_urls in urls_per_manager.items():
+                if not manager_urls:
+                    continue
+                pages = await manager.get_pages(CrawlerRunConfig(), count=len(manager_urls))
+                all_pages.extend(zip(pages, manager_urls))
+            
+            # Crawl pages with delay to avoid rate limiting
+            async def crawl_page(page_ctx, url):
+                page, _ = page_ctx
+                try:
+                    await page.goto(url)
+                    if rate_limit_delay > 0:
+                        await asyncio.sleep(rate_limit_delay)
+                    title = await page.title()
+                    return title
+                finally:
+                    await page.close()
+            
+            crawl_start = time.time()
+            crawl_tasks = [crawl_page(page_ctx, url) for page_ctx, url in all_pages]
+            await asyncio.gather(*crawl_tasks)
+            crawl_time = time.time() - crawl_start
+            total_time = time.time() - start_time
+            
+            # Measure final memory usage
+            if start_memory > 0:
+                end_memory = process.memory_info().rss / (1024 * 1024)
+                memory_used = end_memory - start_memory
+            else:
+                memory_used = 0
+            
+            # Close all browsers
+            for manager in managers:
+                await manager.close()
+            
+            # Calculate metrics
+            pages_per_second = total_urls / crawl_time
+            
+            # Calculate efficiency score (higher is better)
+            # This balances speed vs memory
+            if memory_used > 0:
+                efficiency = pages_per_second / (memory_used + 1)
+            else:
+                efficiency = pages_per_second
+            
+            # Store result
+            result = {
+                "browser_count": browser_count,
+                "distribution": tuple(page_distribution),
+                "crawl_time": crawl_time,
+                "total_time": total_time,
+                "memory_used": memory_used,
+                "pages_per_second": pages_per_second, 
+                "efficiency": efficiency
+            }
+            
+            results.append(result)
+            
+            if verbose:
+                print(f"  ✓ Crawled {total_urls} pages in {crawl_time:.2f}s ({pages_per_second:.1f} pages/sec)")
+                if memory_used > 0:
+                    print(f"  ✓ Memory used: {memory_used:.1f}MB ({memory_used/total_urls:.1f}MB per page)")
+                print(f"  ✓ Efficiency score: {efficiency:.4f}")
+            
+        except Exception as e:
+            if verbose:
+                print(f"  ✗ Error: {str(e)}")
+            
+            # Clean up
+            for manager in managers:
+                try:
+                    await manager.close()
+                except:
+                    pass
+    
+    # If no successful results, return None
+    if not results:
+        return None
+    
+    # Find best configurations
+    fastest = sorted(results, key=lambda x: x["crawl_time"])[0]
+    
+    # Only consider memory if available
+    memory_results = [r for r in results if r["memory_used"] > 0]
+    if memory_results:
+        lowest_memory = sorted(memory_results, key=lambda x: x["memory_used"])[0]
+    else:
+        lowest_memory = fastest
+    
+    # Find most efficient (balanced speed vs memory)
+    optimal = sorted(results, key=lambda x: x["efficiency"], reverse=True)[0]
+    
+    # Print summary
+    if verbose:
+        print("\n=== OPTIMAL CONFIGURATIONS ===")
+        print(f"⚡ Fastest: {fastest['browser_count']} browsers {fastest['distribution']}")
+        print(f"   {fastest['crawl_time']:.2f}s, {fastest['pages_per_second']:.1f} pages/sec")
+        
+        print(f"💾 Memory-efficient: {lowest_memory['browser_count']} browsers {lowest_memory['distribution']}")
+        if lowest_memory["memory_used"] > 0:
+            print(f"   {lowest_memory['memory_used']:.1f}MB, {lowest_memory['memory_used']/total_urls:.2f}MB per page")
+        
+        print(f"🌟 Balanced optimal: {optimal['browser_count']} browsers {optimal['distribution']}")
+        print(f"   {optimal['crawl_time']:.2f}s, {optimal['pages_per_second']:.1f} pages/sec, score: {optimal['efficiency']:.4f}")
+    
+    return {
+        "fastest": fastest,
+        "lowest_memory": lowest_memory,
+        "optimal": optimal,
+        "all_configs": results
+    }

crawl4ai/browser_manager.py

@@ -440,7 +440,8 @@ class BrowserManager:
     @classmethod
     async def get_playwright(cls):
         from playwright.async_api import async_playwright
-        cls._playwright_instance = await async_playwright().start()
+        if cls._playwright_instance is None:
+            cls._playwright_instance = await async_playwright().start()
         return cls._playwright_instance    
 
     def __init__(self, browser_config: BrowserConfig, logger=None):
@@ -491,12 +492,11 @@ class BrowserManager:
 
         Note: This method should be called in a separate task to avoid blocking the main event loop.
         """
-        if self.playwright is not None:
-            await self.close()
-            
-        from playwright.async_api import async_playwright
+        self.playwright  = await self.get_playwright()
+        if self.playwright is None:
+            from playwright.async_api import async_playwright
 
-        self.playwright = await async_playwright().start()
+            self.playwright = await async_playwright().start()
 
         if self.config.cdp_url or self.config.use_managed_browser:
             self.config.use_managed_browser = True
@@ -660,7 +660,7 @@ class BrowserManager:
                     "name": "cookiesEnabled",
                     "value": "true",
                     "url": crawlerRunConfig.url
-                    if crawlerRunConfig and crawlerRunConfig.url
+                    if crawlerRunConfig
                     else "https://crawl4ai.com/",
                 }
             ]

crawl4ai/cli.py

@@ -20,16 +20,13 @@ from crawl4ai import (
     BrowserConfig, 
     CrawlerRunConfig,
     LLMExtractionStrategy, 
-    LXMLWebScrapingStrategy,
     JsonCssExtractionStrategy,
     JsonXPathExtractionStrategy,
     BM25ContentFilter, 
     PruningContentFilter,
     BrowserProfiler,
-    DefaultMarkdownGenerator,
     LLMConfig
 )
-from crawl4ai.config import USER_SETTINGS
 from litellm import completion
 from pathlib import Path
 
@@ -178,12 +175,8 @@ def show_examples():
     # CSS-based extraction
     crwl https://example.com -e extract_css.yml -s css_schema.json -o json
 
-    # LLM-based extraction with config file
+    # LLM-based extraction
     crwl https://example.com -e extract_llm.yml -s llm_schema.json -o json
-    
-    # Quick LLM-based JSON extraction (prompts for LLM provider first time)
-    crwl https://example.com -j  # Auto-extracts structured data
-    crwl https://example.com -j "Extract product details including name, price, and features"  # With specific instructions
 
 3️⃣  Direct Parameters:
     # Browser settings
@@ -285,7 +278,7 @@ llm_schema.json:
     # Combine configs with direct parameters
     crwl https://example.com -B browser.yml -b "headless=false,viewport_width=1920"
 
-    # Full extraction pipeline with config files
+    # Full extraction pipeline
     crwl https://example.com \\
         -B browser.yml \\
         -C crawler.yml \\
@@ -293,12 +286,6 @@ llm_schema.json:
         -s llm_schema.json \\
         -o json \\
         -v
-        
-    # Quick LLM-based extraction with specific instructions
-    crwl https://amazon.com/dp/B01DFKC2SO \\
-        -j "Extract product title, current price, original price, rating, and all product specifications" \\
-        -b "headless=true,viewport_width=1280" \\
-        -v
 
     # Content filtering with BM25
     crwl https://example.com \\
@@ -340,14 +327,6 @@ For more documentation visit: https://github.com/unclecode/crawl4ai
       - google/gemini-pro
     
     See full list of providers: https://docs.litellm.ai/docs/providers
-    
-    # Set default LLM provider and token in advance
-    crwl config set DEFAULT_LLM_PROVIDER "anthropic/claude-3-sonnet"
-    crwl config set DEFAULT_LLM_PROVIDER_TOKEN "your-api-token-here"
-    
-    # Set default browser behavior
-    crwl config set BROWSER_HEADLESS false  # Always show browser window
-    crwl config set USER_AGENT_MODE random  # Use random user agent
 
 9️⃣ Profile Management:
     # Launch interactive profile manager
@@ -1004,19 +983,17 @@ def cdp_cmd(user_data_dir: Optional[str], port: int, browser_type: str, headless
 @click.option("--crawler-config", "-C", type=click.Path(exists=True), help="Crawler config file (YAML/JSON)")
 @click.option("--filter-config", "-f", type=click.Path(exists=True), help="Content filter config file")
 @click.option("--extraction-config", "-e", type=click.Path(exists=True), help="Extraction strategy config file")
-@click.option("--json-extract", "-j", is_flag=False, flag_value="", default=None, help="Extract structured data using LLM with optional description")
 @click.option("--schema", "-s", type=click.Path(exists=True), help="JSON schema for extraction")
 @click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2")
 @click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2")
 @click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all")
-@click.option("--output-file", "-O", type=click.Path(), help="Output file path (default: stdout)")
-@click.option("--bypass-cache", "-b", is_flag=True, default=True, help="Bypass cache when crawling")
+@click.option("--bypass-cache", is_flag=True, default=True, help="Bypass cache when crawling")
 @click.option("--question", "-q", help="Ask a question about the crawled content")
 @click.option("--verbose", "-v", is_flag=True)
 @click.option("--profile", "-p", help="Use a specific browser profile (by name)")
 def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config: str, 
-           extraction_config: str, json_extract: str, schema: str, browser: Dict, crawler: Dict,
-           output: str, output_file: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
+           extraction_config: str, schema: str, browser: Dict, crawler: Dict,
+           output: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
     """Crawl a website and extract content
     
     Simple Usage:
@@ -1060,65 +1037,21 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
             crawler_cfg = crawler_cfg.clone(**crawler)
             
         # Handle content filter config
-        if filter_config or output in ["markdown-fit", "md-fit"]:
-            if filter_config:
-                filter_conf = load_config_file(filter_config)
-            elif not filter_config and output in ["markdown-fit", "md-fit"]:
-                filter_conf = {
-                    "type": "pruning",
-                    "query": "",
-                    "threshold": 0.48
-                }
+        if filter_config:
+            filter_conf = load_config_file(filter_config)
             if filter_conf["type"] == "bm25":
-                crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
-                    content_filter = BM25ContentFilter(
-                        user_query=filter_conf.get("query"),
-                        bm25_threshold=filter_conf.get("threshold", 1.0)
-                    )
+                crawler_cfg.content_filter = BM25ContentFilter(
+                    user_query=filter_conf.get("query"),
+                    bm25_threshold=filter_conf.get("threshold", 1.0)
                 )
             elif filter_conf["type"] == "pruning":
-                crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
-                    content_filter = PruningContentFilter(
-                        user_query=filter_conf.get("query"),
-                        threshold=filter_conf.get("threshold", 0.48)
-                    )
+                crawler_cfg.content_filter = PruningContentFilter(
+                    user_query=filter_conf.get("query"),
+                    threshold=filter_conf.get("threshold", 0.48)
                 )
-        
-        # Handle json-extract option (takes precedence over extraction-config)
-        if json_extract is not None:
-            # Get LLM provider and token
-            provider, token = setup_llm_config()
-            
-            # Default sophisticated instruction for structured data extraction
-            default_instruction = """Analyze the web page content and extract structured data as JSON. 
-If the page contains a list of items with repeated patterns, extract all items in an array. 
-If the page is an article or contains unique content, extract a comprehensive JSON object with all relevant information.
-Look at the content, intention of content, what it offers and find the data item(s) in the page.
-Always return valid, properly formatted JSON."""
-            
-            
-            default_instruction_with_user_query = """Analyze the web page content and extract structured data as JSON, following the below instruction and explanation of schema and always return valid, properly formatted JSON. \n\nInstruction:\n\n""" + json_extract
-            
-            # Determine instruction based on whether json_extract is empty or has content
-            instruction = default_instruction_with_user_query if json_extract else default_instruction
-            
-            # Create LLM extraction strategy
-            crawler_cfg.extraction_strategy = LLMExtractionStrategy(
-                llm_config=LLMConfig(provider=provider, api_token=token),
-                instruction=instruction,
-                schema=load_schema_file(schema),  # Will be None if no schema is provided
-                extraction_type="schema", #if schema else "block",
-                apply_chunking=False,
-                force_json_response=True,
-                verbose=verbose,
-            )
-            
-            # Set output to JSON if not explicitly specified
-            if output == "all":
-                output = "json"
                 
-        # Handle extraction strategy from config file (only if json-extract wasn't used)
-        elif extraction_config:
+        # Handle extraction strategy
+        if extraction_config:
             extract_conf = load_config_file(extraction_config)
             schema_data = load_schema_file(schema)
             
@@ -1152,13 +1085,6 @@ Always return valid, properly formatted JSON."""
         # No cache
         if bypass_cache:
             crawler_cfg.cache_mode = CacheMode.BYPASS
-
-        crawler_cfg.scraping_strategy = LXMLWebScrapingStrategy()    
-
-        config = get_global_config()
-        
-        browser_cfg.verbose = config.get("VERBOSE", False)
-        crawler_cfg.verbose = config.get("VERBOSE", False)
         
         # Run crawler
         result : CrawlResult = anyio.run(
@@ -1177,31 +1103,14 @@ Always return valid, properly formatted JSON."""
             return
         
         # Handle output
-        if not output_file:
-            if output == "all":
-                click.echo(json.dumps(result.model_dump(), indent=2))
-            elif output == "json":
-                print(result.extracted_content)
-                extracted_items = json.loads(result.extracted_content)
-                click.echo(json.dumps(extracted_items, indent=2))
-                
-            elif output in ["markdown", "md"]:
-                click.echo(result.markdown.raw_markdown)
-            elif output in ["markdown-fit", "md-fit"]:
-                click.echo(result.markdown.fit_markdown)
-        else:
-            if output == "all":
-                with open(output_file, "w") as f:
-                    f.write(json.dumps(result.model_dump(), indent=2))
-            elif output == "json":
-                with open(output_file, "w") as f:
-                    f.write(result.extracted_content)
-            elif output in ["markdown", "md"]:
-                with open(output_file, "w") as f:
-                    f.write(result.markdown.raw_markdown)
-            elif output in ["markdown-fit", "md-fit"]:
-                with open(output_file, "w") as f:
-                    f.write(result.markdown.fit_markdown)
+        if output == "all":
+            click.echo(json.dumps(result.model_dump(), indent=2))
+        elif output == "json":
+            click.echo(json.dumps(json.loads(result.extracted_content), indent=2))
+        elif output in ["markdown", "md"]:
+            click.echo(result.markdown.raw_markdown)
+        elif output in ["markdown-fit", "md-fit"]:
+            click.echo(result.markdown.fit_markdown)
             
     except Exception as e:
         raise click.ClickException(str(e))
@@ -1211,120 +1120,6 @@ def examples_cmd():
     """Show usage examples"""
     show_examples()
 
-@cli.group("config")
-def config_cmd():
-    """Manage global configuration settings
-    
-    Commands to view and update global configuration settings:
-    - list: Display all current configuration settings
-    - get: Get the value of a specific setting
-    - set: Set the value of a specific setting
-    """
-    pass
-
-@config_cmd.command("list")
-def config_list_cmd():
-    """List all configuration settings"""
-    config = get_global_config()
-    
-    table = Table(title="Crawl4AI Configuration", show_header=True, header_style="bold cyan", border_style="blue")
-    table.add_column("Setting", style="cyan")
-    table.add_column("Value", style="green")
-    table.add_column("Default", style="yellow")
-    table.add_column("Description", style="white")
-    
-    for key, setting in USER_SETTINGS.items():
-        value = config.get(key, setting["default"])
-        
-        # Handle secret values
-        display_value = value
-        if setting.get("secret", False) and value:
-            display_value = "********"
-            
-        # Handle boolean values
-        if setting["type"] == "boolean":
-            display_value = str(value).lower()
-            default_value = str(setting["default"]).lower()
-        else:
-            default_value = str(setting["default"])
-        
-        table.add_row(
-            key,
-            str(display_value),
-            default_value,
-            setting["description"]
-        )
-    
-    console.print(table)
-
-@config_cmd.command("get")
-@click.argument("key", required=True)
-def config_get_cmd(key: str):
-    """Get a specific configuration setting"""
-    config = get_global_config()
-    
-    # Normalize key to uppercase
-    key = key.upper()
-    
-    if key not in USER_SETTINGS:
-        console.print(f"[red]Error: Unknown setting '{key}'[/red]")
-        return
-    
-    value = config.get(key, USER_SETTINGS[key]["default"])
-    
-    # Handle secret values
-    display_value = value
-    if USER_SETTINGS[key].get("secret", False) and value:
-        display_value = "********"
-    
-    console.print(f"[cyan]{key}[/cyan] = [green]{display_value}[/green]")
-    console.print(f"[dim]Description: {USER_SETTINGS[key]['description']}[/dim]")
-
-@config_cmd.command("set")
-@click.argument("key", required=True)
-@click.argument("value", required=True)
-def config_set_cmd(key: str, value: str):
-    """Set a configuration setting"""
-    config = get_global_config()
-    
-    # Normalize key to uppercase
-    key = key.upper()
-    
-    if key not in USER_SETTINGS:
-        console.print(f"[red]Error: Unknown setting '{key}'[/red]")
-        console.print(f"[yellow]Available settings: {', '.join(USER_SETTINGS.keys())}[/yellow]")
-        return
-    
-    setting = USER_SETTINGS[key]
-    
-    # Type conversion and validation
-    if setting["type"] == "boolean":
-        if value.lower() in ["true", "yes", "1", "y"]:
-            typed_value = True
-        elif value.lower() in ["false", "no", "0", "n"]:
-            typed_value = False
-        else:
-            console.print(f"[red]Error: Invalid boolean value. Use 'true' or 'false'.[/red]")
-            return
-    elif setting["type"] == "string":
-        typed_value = value
-        
-        # Check if the value should be one of the allowed options
-        if "options" in setting and value not in setting["options"]:
-            console.print(f"[red]Error: Value must be one of: {', '.join(setting['options'])}[/red]")
-            return
-    
-    # Update config
-    config[key] = typed_value
-    save_global_config(config)
-    
-    # Handle secret values for display
-    display_value = typed_value
-    if setting.get("secret", False) and typed_value:
-        display_value = "********"
-        
-    console.print(f"[green]Successfully set[/green] [cyan]{key}[/cyan] = [green]{display_value}[/green]")
-
 @cli.command("profiles")
 def profiles_cmd():
     """Manage browser profiles interactively
@@ -1344,7 +1139,6 @@ def profiles_cmd():
 @click.option("--crawler-config", "-C", type=click.Path(exists=True), help="Crawler config file (YAML/JSON)")
 @click.option("--filter-config", "-f", type=click.Path(exists=True), help="Content filter config file")
 @click.option("--extraction-config", "-e", type=click.Path(exists=True), help="Extraction strategy config file")
-@click.option("--json-extract", "-j", is_flag=False, flag_value="", default=None, help="Extract structured data using LLM with optional description")
 @click.option("--schema", "-s", type=click.Path(exists=True), help="JSON schema for extraction")
 @click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2")
 @click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2")
@@ -1354,7 +1148,7 @@ def profiles_cmd():
 @click.option("--verbose", "-v", is_flag=True)
 @click.option("--profile", "-p", help="Use a specific browser profile (by name)")
 def default(url: str, example: bool, browser_config: str, crawler_config: str, filter_config: str, 
-        extraction_config: str, json_extract: str, schema: str, browser: Dict, crawler: Dict,
+        extraction_config: str, schema: str, browser: Dict, crawler: Dict,
         output: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
     """Crawl4AI CLI - Web content extraction tool
 
@@ -1368,14 +1162,7 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
         crwl crawl      - Crawl a website with advanced options
         crwl cdp        - Launch browser with CDP debugging enabled
         crwl browser    - Manage builtin browser (start, stop, status, restart)
-        crwl config     - Manage global configuration settings
         crwl examples   - Show more usage examples
-        
-    Configuration Examples:
-        crwl config list                         - List all configuration settings
-        crwl config get DEFAULT_LLM_PROVIDER     - Show current LLM provider
-        crwl config set VERBOSE true             - Enable verbose mode globally
-        crwl config set BROWSER_HEADLESS false   - Default to visible browser
     """
 
     if example:
@@ -1396,8 +1183,7 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
         browser_config=browser_config,
         crawler_config=crawler_config,
         filter_config=filter_config,
-        extraction_config=extraction_config,
-        json_extract=json_extract,
+        extraction_config=extraction_config, 
         schema=schema,
         browser=browser,
         crawler=crawler,

crawl4ai/config.py

@@ -93,46 +93,3 @@ SHOW_DEPRECATION_WARNINGS = True
 SCREENSHOT_HEIGHT_TRESHOLD = 10000
 PAGE_TIMEOUT = 60000
 DOWNLOAD_PAGE_TIMEOUT = 60000
-
-# Global user settings with descriptions and default values
-USER_SETTINGS = {
-    "DEFAULT_LLM_PROVIDER": {
-        "default": "openai/gpt-4o",
-        "description": "Default LLM provider in 'company/model' format (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')",
-        "type": "string"
-    },
-    "DEFAULT_LLM_PROVIDER_TOKEN": {
-        "default": "",
-        "description": "API token for the default LLM provider",
-        "type": "string",
-        "secret": True
-    },
-    "VERBOSE": {
-        "default": False,
-        "description": "Enable verbose output for all commands",
-        "type": "boolean"
-    },
-    "BROWSER_HEADLESS": {
-        "default": True,
-        "description": "Run browser in headless mode by default",
-        "type": "boolean"
-    },
-    "BROWSER_TYPE": {
-        "default": "chromium",
-        "description": "Default browser type (chromium or firefox)",
-        "type": "string",
-        "options": ["chromium", "firefox"]
-    },
-    "CACHE_MODE": {
-        "default": "bypass",
-        "description": "Default cache mode (bypass, use, or refresh)",
-        "type": "string",
-        "options": ["bypass", "use", "refresh"]
-    },
-    "USER_AGENT_MODE": {
-        "default": "default",
-        "description": "Default user agent mode (default, random, or mobile)",
-        "type": "string",
-        "options": ["default", "random", "mobile"]
-    }
-}

crawl4ai/content_scraping_strategy.py

@@ -860,12 +860,6 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         soup = BeautifulSoup(html, parser_type)
         body = soup.body
         base_domain = get_base_domain(url)
-        
-        # Early removal of all images if exclude_all_images is set
-        # This happens before any processing to minimize memory usage
-        if kwargs.get("exclude_all_images", False):
-            for img in body.find_all('img'):
-                img.decompose()
 
         try:
             meta = extract_metadata("", soup)
@@ -1497,13 +1491,6 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
             body = doc
 
             base_domain = get_base_domain(url)
-            
-            # Early removal of all images if exclude_all_images is set
-            # This is more efficient in lxml as we remove elements before any processing
-            if kwargs.get("exclude_all_images", False):
-                for img in body.xpath('//img'):
-                    if img.getparent() is not None:
-                        img.getparent().remove(img)
 
             # Add comment removal
             if kwargs.get("remove_comments", False):

crawl4ai/deep_crawling/base_strategy.py

@@ -7,6 +7,7 @@ from contextvars import ContextVar
 from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
 
 
+
 class DeepCrawlDecorator:
     """Decorator that adds deep crawling capability to arun method."""
     deep_crawl_active = ContextVar("deep_crawl_active", default=False)
@@ -59,7 +60,8 @@ class DeepCrawlStrategy(ABC):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
+    # ) -> List[CrawlResult]:
+    ) -> RunManyReturn:
         """
         Batch (non-streaming) mode:
         Processes one BFS level at a time, then yields all the results.
@@ -72,7 +74,8 @@ class DeepCrawlStrategy(ABC):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
+    # ) -> AsyncGenerator[CrawlResult, None]:
+    ) -> RunManyReturn:
         """
         Streaming mode:
         Processes one BFS level at a time and yields results immediately as they arrive.

crawl4ai/deep_crawling/bfs_strategy.py

@@ -9,7 +9,7 @@ from ..models import TraversalStats
 from .filters import FilterChain
 from .scorers import URLScorer
 from . import DeepCrawlStrategy  
-from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
 from ..utils import normalize_url_for_deep_crawl, efficient_normalize_url_for_deep_crawl
 from math import inf as infinity
 
@@ -143,7 +143,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
+    # ) -> List[CrawlResult]:
+    ) -> RunManyReturn:
         """
         Batch (non-streaming) mode:
         Processes one BFS level at a time, then yields all the results.
@@ -191,7 +192,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
+    # ) -> AsyncGenerator[CrawlResult, None]:
+    ) -> RunManyReturn:
         """
         Streaming mode:
         Processes one BFS level at a time and yields results immediately as they arrive.

crawl4ai/deep_crawling/dfs_strategy.py

@@ -3,7 +3,7 @@ from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
 
 from ..models import CrawlResult
 from .bfs_strategy import BFSDeepCrawlStrategy  # noqa
-from ..types import AsyncWebCrawler, CrawlerRunConfig
+from ..types import AsyncWebCrawler, CrawlerRunConfig, RunManyReturn
 
 class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
     """
@@ -17,7 +17,8 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
+    # ) -> List[CrawlResult]:
+    ) -> RunManyReturn:
         """
         Batch (non-streaming) DFS mode.
         Uses a stack to traverse URLs in DFS order, aggregating CrawlResults into a list.
@@ -65,7 +66,8 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
         start_url: str,
         crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
+    # ) -> AsyncGenerator[CrawlResult, None]:
+    ) -> RunManyReturn:
         """
         Streaming DFS mode.
         Uses a stack to traverse URLs in DFS order and yields CrawlResults as they become available.

crawl4ai/extraction_strategy.py

@@ -5,11 +5,9 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 import json
 import time
 
-from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH, PROMPT_EXTRACT_INFERRED_SCHEMA
+from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH
 from .config import (
-    DEFAULT_PROVIDER,
-    DEFAULT_PROVIDER_API_KEY,
-    CHUNK_TOKEN_THRESHOLD,
+    DEFAULT_PROVIDER, CHUNK_TOKEN_THRESHOLD,
     OVERLAP_RATE,
     WORD_TOKEN_RATE,
 )
@@ -509,7 +507,6 @@ class LLMExtractionStrategy(ExtractionStrategy):
         word_token_rate=WORD_TOKEN_RATE,
         apply_chunking=True,
         input_format: str = "markdown",
-        force_json_response=False,
         verbose=False,
         # Deprecated arguments
         provider: str = DEFAULT_PROVIDER,
@@ -530,10 +527,9 @@ class LLMExtractionStrategy(ExtractionStrategy):
             overlap_rate: Overlap between chunks.
             word_token_rate: Word to token conversion rate.
             apply_chunking: Whether to apply chunking.
-            input_format: Content format to use for extraction.
-                            Options: "markdown" (default), "html", "fit_markdown"
-            force_json_response: Whether to force a JSON response from the LLM.
             verbose: Whether to print verbose output.
+            usages: List of individual token usages.
+            total_usage: Accumulated token usage.
 
             # Deprecated arguments, will be removed very soon
             provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
@@ -544,17 +540,11 @@ class LLMExtractionStrategy(ExtractionStrategy):
         """
         super().__init__( input_format=input_format, **kwargs)
         self.llm_config = llm_config
-        if not self.llm_config:
-            self.llm_config = create_llm_config(
-                provider=DEFAULT_PROVIDER,
-                api_token=os.environ.get(DEFAULT_PROVIDER_API_KEY),
-            )
         self.instruction = instruction
         self.extract_type = extraction_type
         self.schema = schema
         if schema:
             self.extract_type = "schema"
-        self.force_json_response = force_json_response
         self.chunk_token_threshold = chunk_token_threshold or CHUNK_TOKEN_THRESHOLD
         self.overlap_rate = overlap_rate
         self.word_token_rate = word_token_rate
@@ -618,97 +608,64 @@ class LLMExtractionStrategy(ExtractionStrategy):
             variable_values["SCHEMA"] = json.dumps(self.schema, indent=2) # if type of self.schema is dict else self.schema
             prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
 
-        if self.extract_type == "schema" and not self.schema:
-            prompt_with_variables = PROMPT_EXTRACT_INFERRED_SCHEMA
-
         for variable in variable_values:
             prompt_with_variables = prompt_with_variables.replace(
                 "{" + variable + "}", variable_values[variable]
             )
 
+        response = perform_completion_with_backoff(
+            self.llm_config.provider,
+            prompt_with_variables,
+            self.llm_config.api_token,
+            base_url=self.llm_config.base_url,
+            extra_args=self.extra_args,
+        )  # , json_response=self.extract_type == "schema")
+        # Track usage
+        usage = TokenUsage(
+            completion_tokens=response.usage.completion_tokens,
+            prompt_tokens=response.usage.prompt_tokens,
+            total_tokens=response.usage.total_tokens,
+            completion_tokens_details=response.usage.completion_tokens_details.__dict__
+            if response.usage.completion_tokens_details
+            else {},
+            prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
+            if response.usage.prompt_tokens_details
+            else {},
+        )
+        self.usages.append(usage)
+
+        # Update totals
+        self.total_usage.completion_tokens += usage.completion_tokens
+        self.total_usage.prompt_tokens += usage.prompt_tokens
+        self.total_usage.total_tokens += usage.total_tokens
+
         try:
-            response = perform_completion_with_backoff(
-                self.llm_config.provider,
-                prompt_with_variables,
-                self.llm_config.api_token,
-                base_url=self.llm_config.base_url,
-                json_response=self.force_json_response,
-                extra_args=self.extra_args,
-            )  # , json_response=self.extract_type == "schema")
-            # Track usage
-            usage = TokenUsage(
-                completion_tokens=response.usage.completion_tokens,
-                prompt_tokens=response.usage.prompt_tokens,
-                total_tokens=response.usage.total_tokens,
-                completion_tokens_details=response.usage.completion_tokens_details.__dict__
-                if response.usage.completion_tokens_details
-                else {},
-                prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
-                if response.usage.prompt_tokens_details
-                else {},
-            )
-            self.usages.append(usage)
-
-            # Update totals
-            self.total_usage.completion_tokens += usage.completion_tokens
-            self.total_usage.prompt_tokens += usage.prompt_tokens
-            self.total_usage.total_tokens += usage.total_tokens
-
-            try:
-                response = response.choices[0].message.content
-                blocks = None
-
-                if self.force_json_response:
-                    blocks = json.loads(response)
-                    if isinstance(blocks, dict):
-                        # If it has only one key which calue is list then assign that to blocks, exampled: {"news": [..]}
-                        if len(blocks) == 1 and isinstance(list(blocks.values())[0], list):
-                            blocks = list(blocks.values())[0]
-                        else:
-                            # If it has only one key which value is not list then assign that to blocks, exampled: { "article_id": "1234", ... }
-                            blocks = [blocks]
-                    elif isinstance(blocks, list):
-                        # If it is a list then assign that to blocks
-                        blocks = blocks
-                else: 
-                    # blocks = extract_xml_data(["blocks"], response.choices[0].message.content)["blocks"]
-                    blocks = extract_xml_data(["blocks"], response)["blocks"]
-                    blocks = json.loads(blocks)
-
-                for block in blocks:
-                    block["error"] = False
-            except Exception:
-                parsed, unparsed = split_and_parse_json_objects(
-                    response.choices[0].message.content
-                )
-                blocks = parsed
-                if unparsed:
-                    blocks.append(
-                        {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
-                    )
-
-            if self.verbose:
-                print(
-                    "[LOG] Extracted",
-                    len(blocks),
-                    "blocks from URL:",
-                    url,
-                    "block index:",
-                    ix,
-                )
-            return blocks
-        except Exception as e:
-            if self.verbose:
-                print(f"[LOG] Error in LLM extraction: {e}")
-            # Add error information to extracted_content
-            return [
-                {
-                    "index": ix,
-                    "error": True,
-                    "tags": ["error"],
-                    "content": str(e),
-                }
+            blocks = extract_xml_data(["blocks"], response.choices[0].message.content)[
+                "blocks"
             ]
+            blocks = json.loads(blocks)
+            for block in blocks:
+                block["error"] = False
+        except Exception:
+            parsed, unparsed = split_and_parse_json_objects(
+                response.choices[0].message.content
+            )
+            blocks = parsed
+            if unparsed:
+                blocks.append(
+                    {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
+                )
+
+        if self.verbose:
+            print(
+                "[LOG] Extracted",
+                len(blocks),
+                "blocks from URL:",
+                url,
+                "block index:",
+                ix,
+            )
+        return blocks
 
     def _merge(self, documents, chunk_token_threshold, overlap) -> List[str]:
         """

crawl4ai/install.py

@@ -40,28 +40,12 @@ def setup_home_directory():
             f.write("")
 
 def post_install():
-    """
-    Run all post-installation tasks.
-    Checks CRAWL4AI_MODE environment variable. If set to 'api',
-    skips Playwright browser installation.
-    """
+    """Run all post-installation tasks"""
     logger.info("Running post-installation setup...", tag="INIT")
     setup_home_directory()
-
-    # Check environment variable to conditionally skip Playwright install
-    run_mode = os.getenv('CRAWL4AI_MODE')
-    if run_mode == 'api':
-        logger.warning(
-            "CRAWL4AI_MODE=api detected. Skipping Playwright browser installation.",
-            tag="SETUP"
-        )
-    else:
-        # Proceed with installation only if mode is not 'api'
-        install_playwright()
-
+    install_playwright()
     run_migration()
-    # TODO: Will be added in the future
-    # setup_builtin_browser()
+    setup_builtin_browser()
     logger.success("Post-installation setup completed!", tag="COMPLETE")
     
 def setup_builtin_browser():

crawl4ai/models.py

@@ -1,7 +1,5 @@
 from pydantic import BaseModel, HttpUrl, PrivateAttr
 from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
-from typing import AsyncGenerator
-from typing import Generic, TypeVar
 from enum import Enum
 from dataclasses import dataclass
 from .ssl_certificate import SSLCertificate
@@ -36,12 +34,34 @@ class CrawlerTaskResult:
     def success(self) -> bool:
         return self.result.success
 
+
 class CrawlStatus(Enum):
     QUEUED = "QUEUED"
     IN_PROGRESS = "IN_PROGRESS"
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
 
+
+# @dataclass
+# class CrawlStats:
+#     task_id: str
+#     url: str
+#     status: CrawlStatus
+#     start_time: Optional[datetime] = None
+#     end_time: Optional[datetime] = None
+#     memory_usage: float = 0.0
+#     peak_memory: float = 0.0
+#     error_message: str = ""
+
+#     @property
+#     def duration(self) -> str:
+#         if not self.start_time:
+#             return "0:00"
+#         end = self.end_time or datetime.now()
+#         duration = end - self.start_time
+#         return str(timedelta(seconds=int(duration.total_seconds())))
+
+
 @dataclass
 class CrawlStats:
     task_id: str
@@ -75,6 +95,7 @@ class CrawlStats:
         duration = end - start
         return str(timedelta(seconds=int(duration.total_seconds())))
 
+
 class DisplayMode(Enum):
     DETAILED = "DETAILED"
     AGGREGATED = "AGGREGATED"
@@ -91,11 +112,21 @@ class TokenUsage:
     completion_tokens_details: Optional[dict] = None
     prompt_tokens_details: Optional[dict] = None
 
+
 class UrlModel(BaseModel):
     url: HttpUrl
     forced: bool = False
 
 
+class MarkdownGenerationResult(BaseModel):
+    raw_markdown: str
+    markdown_with_citations: str
+    references_markdown: str
+    fit_markdown: Optional[str] = None
+    fit_html: Optional[str] = None
+
+    def __str__(self):
+        return self.raw_markdown
 
 @dataclass
 class TraversalStats:
@@ -116,16 +147,6 @@ class DispatchResult(BaseModel):
     end_time: Union[datetime, float]
     error_message: str = ""
 
-class MarkdownGenerationResult(BaseModel):
-    raw_markdown: str
-    markdown_with_citations: str
-    references_markdown: str
-    fit_markdown: Optional[str] = None
-    fit_html: Optional[str] = None
-
-    def __str__(self):
-        return self.raw_markdown
-    
 class CrawlResult(BaseModel):
     url: str
     html: str
@@ -137,7 +158,6 @@ class CrawlResult(BaseModel):
     js_execution_result: Optional[Dict[str, Any]] = None
     screenshot: Optional[str] = None
     pdf: Optional[bytes] = None
-    mhtml: Optional[str] = None
     _markdown: Optional[MarkdownGenerationResult] = PrivateAttr(default=None)
     extracted_content: Optional[str] = None
     metadata: Optional[dict] = None
@@ -148,8 +168,6 @@ class CrawlResult(BaseModel):
     ssl_certificate: Optional[SSLCertificate] = None
     dispatch_result: Optional[DispatchResult] = None
     redirected_url: Optional[str] = None
-    network_requests: Optional[List[Dict[str, Any]]] = None
-    console_messages: Optional[List[Dict[str, Any]]] = None
 
     class Config:
         arbitrary_types_allowed = True
@@ -266,40 +284,6 @@ class StringCompatibleMarkdown(str):
     def __getattr__(self, name):
         return getattr(self._markdown_result, name)
 
-CrawlResultT = TypeVar('CrawlResultT', bound=CrawlResult)
-
-class CrawlResultContainer(Generic[CrawlResultT]):
-    def __init__(self, results: Union[CrawlResultT, List[CrawlResultT]]):
-        # Normalize to a list
-        if isinstance(results, list):
-            self._results = results
-        else:
-            self._results = [results]
-
-    def __iter__(self):
-        return iter(self._results)
-
-    def __getitem__(self, index):
-        return self._results[index]
-
-    def __len__(self):
-        return len(self._results)
-
-    def __getattr__(self, attr):
-        # Delegate attribute access to the first element.
-        if self._results:
-            return getattr(self._results[0], attr)
-        raise AttributeError(f"{self.__class__.__name__} object has no attribute '{attr}'")
-
-    def __repr__(self):
-        return f"{self.__class__.__name__}({self._results!r})"
-
-RunManyReturn = Union[
-    CrawlResultContainer[CrawlResultT],
-    AsyncGenerator[CrawlResultT, None]
-]
-
-
 # END of backward compatibility code for markdown/markdown_v2.
 # When removing this code in the future, make sure to:
 # 1. Replace the private attribute and property with a standard field
@@ -312,17 +296,15 @@ class AsyncCrawlResponse(BaseModel):
     status_code: int
     screenshot: Optional[str] = None
     pdf_data: Optional[bytes] = None
-    mhtml_data: Optional[str] = None
     get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
     downloaded_files: Optional[List[str]] = None
     ssl_certificate: Optional[SSLCertificate] = None
     redirected_url: Optional[str] = None
-    network_requests: Optional[List[Dict[str, Any]]] = None
-    console_messages: Optional[List[Dict[str, Any]]] = None
 
     class Config:
         arbitrary_types_allowed = True
 
+
 ###############################
 # Scraping Models
 ###############################

crawl4ai/prompts.py

@@ -203,62 +203,6 @@ Avoid Common Mistakes:
 Result
 Output the final list of JSON objects, wrapped in <blocks>...</blocks> XML tags. Make sure to close the tag properly."""
 
-PROMPT_EXTRACT_INFERRED_SCHEMA = """Here is the content from the URL:
-<url>{URL}</url>
-
-<url_content>
-{HTML}
-</url_content>
-
-Please carefully read the URL content and the user's request. Analyze the page structure and infer the most appropriate JSON schema based on the content and request.
-
-Extraction Strategy:
-1. First, determine if the page contains repetitive items (like multiple products, articles, etc.) or a single content item (like a single article or page).
-2. For repetitive items: Identify the common pattern and extract each instance as a separate JSON object in an array.
-3. For single content: Extract the key information into a comprehensive JSON object that captures the essential details.
-
-Extraction instructions:
-Return the extracted information as a list of JSON objects. For repetitive content, each object in the list should correspond to a distinct item. For single content, you may return just one detailed JSON object. Wrap the entire JSON list in <blocks>...</blocks> XML tags.
-
-Schema Design Guidelines:
-- Create meaningful property names that clearly describe the data they contain
-- Use nested objects for hierarchical information
-- Use arrays for lists of related items
-- Include all information requested by the user
-- Maintain consistency in property names and data structures
-- Only include properties that are actually present in the content
-- For dates, prefer ISO format (YYYY-MM-DD)
-- For prices or numeric values, extract them without currency symbols when possible
-
-Quality Reflection:
-Before outputting your final answer, double check that:
-1. The inferred schema makes logical sense for the type of content
-2. All requested information is included
-3. The JSON is valid and could be parsed without errors
-4. Property names are consistent and descriptive
-5. The structure is optimal for the type of data being represented
-
-Avoid Common Mistakes:
-- Do NOT add any comments using "//" or "#" in the JSON output. It causes parsing errors.
-- Make sure the JSON is properly formatted with curly braces, square brackets, and commas in the right places.
-- Do not miss closing </blocks> tag at the end of the JSON output.
-- Do not generate Python code showing how to do the task; this is your task to extract the information and return it in JSON format.
-- Ensure consistency in property names across all objects
-- Don't include empty properties or null values unless they're meaningful
-- For repetitive content, ensure all objects follow the same schema
-
-Important: If user specific instruction is provided, then stress significantly on what user is requesting and describing about the schema of end result (if any). If user is requesting to extract specific information, then focus on that and ignore the rest of the content.
-<user_request>
-{REQUEST}
-</user_request>
-
-Result:
-Output the final list of JSON objects, wrapped in <blocks>...</blocks> XML tags. Make sure to close the tag properly.
-
-DO NOT ADD ANY PRE OR POST COMMENTS. JUST RETURN THE JSON OBJECTS INSIDE <blocks>...</blocks> TAGS.
-
-CRITICAL: The content inside the <blocks> tags MUST be a direct array of JSON objects (starting with '[' and ending with ']'), not a dictionary/object containing an array. For example, use <blocks>[{...}, {...}]</blocks> instead of <blocks>{"items": [{...}, {...}]}</blocks>. This is essential for proper parsing.
-"""
 
 PROMPT_FILTER_CONTENT = """Your task is to filter and convert HTML content into clean, focused markdown that's optimized for use with LLMs and information retrieval systems.
 

crawl4ai/utils.py

@@ -1551,7 +1551,7 @@ def extract_xml_tags(string):
     return list(set(tags))
 
 
-def extract_xml_data_legacy(tags, string):
+def extract_xml_data(tags, string):
     """
     Extract data for specified XML tags from a string.
 
@@ -1580,38 +1580,6 @@ def extract_xml_data_legacy(tags, string):
 
     return data
 
-def extract_xml_data(tags, string):
-    """
-    Extract data for specified XML tags from a string, returning the longest content for each tag.
-
-    How it works:
-    1. Finds all occurrences of each tag in the string using regex.
-    2. For each tag, selects the occurrence with the longest content.
-    3. Returns a dictionary of tag-content pairs.
-
-    Args:
-        tags (List[str]): The list of XML tags to extract.
-        string (str): The input string containing XML data.
-
-    Returns:
-        Dict[str, str]: A dictionary with tag names as keys and longest extracted content as values.
-    """
-
-    data = {}
-
-    for tag in tags:
-        pattern = f"<{tag}>(.*?)</{tag}>"
-        matches = re.findall(pattern, string, re.DOTALL)
-        
-        if matches:
-            # Find the longest content for this tag
-            longest_content = max(matches, key=len).strip()
-            data[tag] = longest_content
-        else:
-            data[tag] = ""
-
-    return data
-
 
 def perform_completion_with_backoff(
     provider,
@@ -1680,19 +1648,6 @@ def perform_completion_with_backoff(
                         "content": ["Rate limit error. Please try again later."],
                     }
                 ]
-        except Exception as e:
-            raise e  # Raise any other exceptions immediately
-            # print("Error during completion request:", str(e))
-            # error_message = e.message
-            # return [
-            #     {
-            #         "index": 0,
-            #         "tags": ["error"],
-            #         "content": [
-            #             f"Error during LLM completion request. {error_message}"
-            #         ],
-            #     }
-            # ]
 
 
 def extract_blocks(url, html, provider=DEFAULT_PROVIDER, api_token=None, base_url=None):

deploy/docker/README-new.md

@@ -1,644 +0,0 @@
-# Crawl4AI Docker Guide 🐳
-
-## Table of Contents
-- [Prerequisites](#prerequisites)
-- [Installation](#installation)
-  - [Option 1: Using Docker Compose (Recommended)](#option-1-using-docker-compose-recommended)
-  - [Option 2: Manual Local Build & Run](#option-2-manual-local-build--run)
-  - [Option 3: Using Pre-built Docker Hub Images](#option-3-using-pre-built-docker-hub-images)
-- [Dockerfile Parameters](#dockerfile-parameters)
-- [Using the API](#using-the-api)
-  - [Understanding Request Schema](#understanding-request-schema)
-  - [REST API Examples](#rest-api-examples)
-  - [Python SDK](#python-sdk)
-- [Metrics & Monitoring](#metrics--monitoring)
-- [Deployment Scenarios](#deployment-scenarios)
-- [Complete Examples](#complete-examples)
-- [Server Configuration](#server-configuration)
-  - [Understanding config.yml](#understanding-configyml)
-  - [JWT Authentication](#jwt-authentication)
-  - [Configuration Tips and Best Practices](#configuration-tips-and-best-practices)
-  - [Customizing Your Configuration](#customizing-your-configuration)
-  - [Configuration Recommendations](#configuration-recommendations)
-- [Getting Help](#getting-help)
-
-## Prerequisites
-
-Before we dive in, make sure you have:
-- Docker installed and running (version 20.10.0 or higher), including `docker compose` (usually bundled with Docker Desktop).
-- `git` for cloning the repository.
-- At least 4GB of RAM available for the container (more recommended for heavy use).
-- Python 3.10+ (if using the Python SDK).
-- Node.js 16+ (if using the Node.js examples).
-
-> 💡 **Pro tip**: Run `docker info` to check your Docker installation and available resources.
-
-## Installation
-
-We offer several ways to get the Crawl4AI server running. Docker Compose is the easiest way to manage local builds and runs.
-
-### Option 1: Using Docker Compose (Recommended)
-
-Docker Compose simplifies building and running the service, especially for local development and testing across different platforms.
-
-#### 1. Clone Repository
-
-```bash
-git clone https://github.com/unclecode/crawl4ai.git
-cd crawl4ai
-```
-
-#### 2. Environment Setup (API Keys)
-
-If you plan to use LLMs, copy the example environment file and add your API keys. This file should be in the **project root directory**.
-
-```bash
-# Make sure you are in the 'crawl4ai' root directory
-cp deploy/docker/.llm.env.example .llm.env
-
-# Now edit .llm.env and add your API keys
-# Example content:
-# OPENAI_API_KEY=sk-your-key
-# ANTHROPIC_API_KEY=your-anthropic-key
-# ...
-```
-> 🔑 **Note**: Keep your API keys secure! Never commit `.llm.env` to version control.
-
-#### 3. Build and Run with Compose
-
-The `docker-compose.yml` file in the project root defines services for different scenarios using **profiles**.
-
-*   **Build and Run Locally (AMD64):**
-    ```bash
-    # Builds the image locally using Dockerfile and runs it
-    docker compose --profile local-amd64 up --build -d
-    ```
-
-*   **Build and Run Locally (ARM64):**
-    ```bash
-    # Builds the image locally using Dockerfile and runs it
-    docker compose --profile local-arm64 up --build -d
-    ```
-
-*   **Run Pre-built Image from Docker Hub (AMD64):**
-    ```bash
-    # Pulls and runs the specified AMD64 image from Docker Hub
-    # (Set VERSION env var for specific tags, e.g., VERSION=0.5.1-d1)
-    docker compose --profile hub-amd64 up -d
-    ```
-
-*   **Run Pre-built Image from Docker Hub (ARM64):**
-    ```bash
-    # Pulls and runs the specified ARM64 image from Docker Hub
-    docker compose --profile hub-arm64 up -d
-    ```
-
-> The server will be available at `http://localhost:11235`.
-
-#### 4. Stopping Compose Services
-
-```bash
-# Stop the service(s) associated with a profile (e.g., local-amd64)
-docker compose --profile local-amd64 down
-```
-
-### Option 2: Manual Local Build & Run
-
-If you prefer not to use Docker Compose for local builds.
-
-#### 1. Clone Repository & Setup Environment
-
-Follow steps 1 and 2 from the Docker Compose section above (clone repo, `cd crawl4ai`, create `.llm.env` in the root).
-
-#### 2. Build the Image (Multi-Arch)
-
-Use `docker buildx` to build the image. This example builds for multiple platforms and loads the image matching your host architecture into the local Docker daemon.
-
-```bash
-# Make sure you are in the 'crawl4ai' root directory
-docker buildx build --platform linux/amd64,linux/arm64 -t crawl4ai-local:latest --load .
-```
-
-#### 3. Run the Container
-
-*   **Basic run (no LLM support):**
-    ```bash
-    # Replace --platform if your host is ARM64
-    docker run -d \
-      -p 11235:11235 \
-      --name crawl4ai-standalone \
-      --shm-size=1g \
-      --platform linux/amd64 \
-      crawl4ai-local:latest
-    ```
-
-*   **With LLM support:**
-    ```bash
-    # Make sure .llm.env is in the current directory (project root)
-    # Replace --platform if your host is ARM64
-    docker run -d \
-      -p 11235:11235 \
-      --name crawl4ai-standalone \
-      --env-file .llm.env \
-      --shm-size=1g \
-      --platform linux/amd64 \
-      crawl4ai-local:latest
-    ```
-
-> The server will be available at `http://localhost:11235`.
-
-#### 4. Stopping the Manual Container
-
-```bash
-docker stop crawl4ai-standalone && docker rm crawl4ai-standalone
-```
-
-### Option 3: Using Pre-built Docker Hub Images
-
-Pull and run images directly from Docker Hub without building locally.
-
-#### 1. Pull the Image
-
-We use a versioning scheme like `LIBRARY_VERSION-dREVISION` (e.g., `0.5.1-d1`). The `latest` tag points to the most recent stable release. Images are built with multi-arch manifests, so Docker usually pulls the correct version for your system automatically.
-
-```bash
-# Pull a specific version (recommended for stability)
-docker pull unclecode/crawl4ai:0.5.1-d1
-
-# Or pull the latest stable version
-docker pull unclecode/crawl4ai:latest
-```
-
-#### 2. Setup Environment (API Keys)
-
-If using LLMs, create the `.llm.env` file in a directory of your choice, similar to Step 2 in the Compose section.
-
-#### 3. Run the Container
-
-*   **Basic run:**
-    ```bash
-    docker run -d \
-      -p 11235:11235 \
-      --name crawl4ai-hub \
-      --shm-size=1g \
-      unclecode/crawl4ai:0.5.1-d1 # Or use :latest
-    ```
-
-*   **With LLM support:**
-    ```bash
-    # Make sure .llm.env is in the current directory you are running docker from
-    docker run -d \
-      -p 11235:11235 \
-      --name crawl4ai-hub \
-      --env-file .llm.env \
-      --shm-size=1g \
-      unclecode/crawl4ai:0.5.1-d1 # Or use :latest
-    ```
-
-> The server will be available at `http://localhost:11235`.
-
-#### 4. Stopping the Hub Container
-
-```bash
-docker stop crawl4ai-hub && docker rm crawl4ai-hub
-```
-
-#### Docker Hub Versioning Explained
-
-*   **Image Name:** `unclecode/crawl4ai`
-*   **Tag Format:** `LIBRARY_VERSION-dREVISION`
-    *   `LIBRARY_VERSION`: The Semantic Version of the core `crawl4ai` Python library included (e.g., `0.5.1`).
-    *   `dREVISION`: An incrementing number (starting at `d1`) for Docker build changes made *without* changing the library version (e.g., base image updates, dependency fixes). Resets to `d1` for each new `LIBRARY_VERSION`.
-*   **Example:** `unclecode/crawl4ai:0.5.1-d1`
-*   **`latest` Tag:** Points to the most recent stable `LIBRARY_VERSION-dREVISION`.
-*   **Multi-Arch:** Images support `linux/amd64` and `linux/arm64`. Docker automatically selects the correct architecture.
-
----
-
-*(Rest of the document remains largely the same, but with key updates below)*
-
----
-
-## Dockerfile Parameters
-
-You can customize the image build process using build arguments (`--build-arg`). These are typically used via `docker buildx build` or within the `docker-compose.yml` file.
-
-```bash
-# Example: Build with 'all' features using buildx
-docker buildx build \
-  --platform linux/amd64,linux/arm64 \
-  --build-arg INSTALL_TYPE=all \
-  -t yourname/crawl4ai-all:latest \
-  --load \
-  . # Build from root context
-```
-
-### Build Arguments Explained
-
-| Argument     | Description                              | Default   | Options                            |
-| :----------- | :--------------------------------------- | :-------- | :--------------------------------- |
-| INSTALL_TYPE | Feature set                              | `default` | `default`, `all`, `torch`, `transformer` |
-| ENABLE_GPU   | GPU support (CUDA for AMD64)           | `false`   | `true`, `false`                    |
-| APP_HOME     | Install path inside container (advanced) | `/app`    | any valid path                   |
-| USE_LOCAL    | Install library from local source        | `true`    | `true`, `false`                    |
-| GITHUB_REPO  | Git repo to clone if USE_LOCAL=false   | *(see Dockerfile)* | any git URL                  |
-| GITHUB_BRANCH| Git branch to clone if USE_LOCAL=false   | `main`    | any branch name                  |
-
-*(Note: PYTHON_VERSION is fixed by the `FROM` instruction in the Dockerfile)*
-
-### Build Best Practices
-
-1.  **Choose the Right Install Type**
-    *   `default`: Basic installation, smallest image size. Suitable for most standard web scraping and markdown generation.
-    *   `all`: Full features including `torch` and `transformers` for advanced extraction strategies (e.g., CosineStrategy, certain LLM filters). Significantly larger image. Ensure you need these extras.
-2.  **Platform Considerations**
-    *   Use `buildx` for building multi-architecture images, especially for pushing to registries.
-    *   Use `docker compose` profiles (`local-amd64`, `local-arm64`) for easy platform-specific local builds.
-3.  **Performance Optimization**
-    *   The image automatically includes platform-specific optimizations (OpenMP for AMD64, OpenBLAS for ARM64).
-
----
-
-## Using the API
-
-Communicate with the running Docker server via its REST API (defaulting to `http://localhost:11235`). You can use the Python SDK or make direct HTTP requests.
-
-### Python SDK
-
-Install the SDK: `pip install crawl4ai`
-
-```python
-import asyncio
-from crawl4ai.docker_client import Crawl4aiDockerClient
-from crawl4ai import BrowserConfig, CrawlerRunConfig, CacheMode # Assuming you have crawl4ai installed
-
-async def main():
-    # Point to the correct server port
-    async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client:
-        # If JWT is enabled on the server, authenticate first:
-        # await client.authenticate("user@example.com") # See Server Configuration section
-
-        # Example Non-streaming crawl
-        print("--- Running Non-Streaming Crawl ---")
-        results = await client.crawl(
-            ["https://httpbin.org/html"],
-            browser_config=BrowserConfig(headless=True), # Use library classes for config aid
-            crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
-        )
-        if results: # client.crawl returns None on failure
-          print(f"Non-streaming results success: {results.success}")
-          if results.success:
-              for result in results: # Iterate through the CrawlResultContainer
-                  print(f"URL: {result.url}, Success: {result.success}")
-        else:
-            print("Non-streaming crawl failed.")
-
-
-        # Example Streaming crawl
-        print("\n--- Running Streaming Crawl ---")
-        stream_config = CrawlerRunConfig(stream=True, cache_mode=CacheMode.BYPASS)
-        try:
-            async for result in await client.crawl( # client.crawl returns an async generator for streaming
-                ["https://httpbin.org/html", "https://httpbin.org/links/5/0"],
-                browser_config=BrowserConfig(headless=True),
-                crawler_config=stream_config
-            ):
-                print(f"Streamed result: URL: {result.url}, Success: {result.success}")
-        except Exception as e:
-            print(f"Streaming crawl failed: {e}")
-
-
-        # Example Get schema
-        print("\n--- Getting Schema ---")
-        schema = await client.get_schema()
-        print(f"Schema received: {bool(schema)}") # Print whether schema was received
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-*(SDK parameters like timeout, verify_ssl etc. remain the same)*
-
-### Second Approach: Direct API Calls
-
-Crucially, when sending configurations directly via JSON, they **must** follow the `{"type": "ClassName", "params": {...}}` structure for any non-primitive value (like config objects or strategies). Dictionaries must be wrapped as `{"type": "dict", "value": {...}}`.
-
-*(Keep the detailed explanation of Configuration Structure, Basic Pattern, Simple vs Complex, Strategy Pattern, Complex Nested Example, Quick Grammar Overview, Important Rules, Pro Tip)*
-
-#### More Examples *(Ensure Schema example uses type/value wrapper)*
-
-**Advanced Crawler Configuration**
-*(Keep example, ensure cache_mode uses valid enum value like "bypass")*
-
-**Extraction Strategy**
-```json
-{
-    "crawler_config": {
-        "type": "CrawlerRunConfig",
-        "params": {
-            "extraction_strategy": {
-                "type": "JsonCssExtractionStrategy",
-                "params": {
-                    "schema": {
-                        "type": "dict",
-                        "value": {
-                           "baseSelector": "article.post",
-                           "fields": [
-                               {"name": "title", "selector": "h1", "type": "text"},
-                               {"name": "content", "selector": ".content", "type": "html"}
-                           ]
-                         }
-                    }
-                }
-            }
-        }
-    }
-}
-```
-
-**LLM Extraction Strategy** *(Keep example, ensure schema uses type/value wrapper)*
-*(Keep Deep Crawler Example)*
-
-### REST API Examples
-
-Update URLs to use port `11235`.
-
-#### Simple Crawl
-
-```python
-import requests
-
-# Configuration objects converted to the required JSON structure
-browser_config_payload = {
-    "type": "BrowserConfig",
-    "params": {"headless": True}
-}
-crawler_config_payload = {
-    "type": "CrawlerRunConfig",
-    "params": {"stream": False, "cache_mode": "bypass"} # Use string value of enum
-}
-
-crawl_payload = {
-    "urls": ["https://httpbin.org/html"],
-    "browser_config": browser_config_payload,
-    "crawler_config": crawler_config_payload
-}
-response = requests.post(
-    "http://localhost:11235/crawl", # Updated port
-    # headers={"Authorization": f"Bearer {token}"},  # If JWT is enabled
-    json=crawl_payload
-)
-print(f"Status Code: {response.status_code}")
-if response.ok:
-    print(response.json())
-else:
-    print(f"Error: {response.text}")
-
-```
-
-#### Streaming Results
-
-```python
-import json
-import httpx # Use httpx for async streaming example
-
-async def test_stream_crawl(token: str = None): # Made token optional
-    """Test the /crawl/stream endpoint with multiple URLs."""
-    url = "http://localhost:11235/crawl/stream" # Updated port
-    payload = {
-        "urls": [
-            "https://httpbin.org/html",
-            "https://httpbin.org/links/5/0",
-        ],
-        "browser_config": {
-            "type": "BrowserConfig",
-            "params": {"headless": True, "viewport": {"type": "dict", "value": {"width": 1200, "height": 800}}} # Viewport needs type:dict
-        },
-        "crawler_config": {
-            "type": "CrawlerRunConfig",
-            "params": {"stream": True, "cache_mode": "bypass"}
-        }
-    }
-
-    headers = {}
-    # if token:
-    #    headers = {"Authorization": f"Bearer {token}"} # If JWT is enabled
-
-    try:
-        async with httpx.AsyncClient() as client:
-            async with client.stream("POST", url, json=payload, headers=headers, timeout=120.0) as response:
-                print(f"Status: {response.status_code} (Expected: 200)")
-                response.raise_for_status() # Raise exception for bad status codes
-
-                # Read streaming response line-by-line (NDJSON)
-                async for line in response.aiter_lines():
-                    if line:
-                        try:
-                            data = json.loads(line)
-                            # Check for completion marker
-                            if data.get("status") == "completed":
-                                print("Stream completed.")
-                                break
-                            print(f"Streamed Result: {json.dumps(data, indent=2)}")
-                        except json.JSONDecodeError:
-                            print(f"Warning: Could not decode JSON line: {line}")
-
-    except httpx.HTTPStatusError as e:
-         print(f"HTTP error occurred: {e.response.status_code} - {e.response.text}")
-    except Exception as e:
-        print(f"Error in streaming crawl test: {str(e)}")
-
-# To run this example:
-# import asyncio
-# asyncio.run(test_stream_crawl())
-```
-
----
-
-## Metrics & Monitoring
-
-Keep an eye on your crawler with these endpoints:
-
-- `/health` - Quick health check
-- `/metrics` - Detailed Prometheus metrics
-- `/schema` - Full API schema
-
-Example health check:
-```bash
-curl http://localhost:11235/health
-```
-
----
-
-*(Deployment Scenarios and Complete Examples sections remain the same, maybe update links if examples moved)*
-
----
-
-## Server Configuration
-
-The server's behavior can be customized through the `config.yml` file.
-
-### Understanding config.yml
-
-The configuration file is loaded from `/app/config.yml` inside the container. By default, the file from `deploy/docker/config.yml` in the repository is copied there during the build.
-
-Here's a detailed breakdown of the configuration options (using defaults from `deploy/docker/config.yml`):
-
-```yaml
-# Application Configuration
-app:
-  title: "Crawl4AI API"
-  version: "1.0.0" # Consider setting this to match library version, e.g., "0.5.1"
-  host: "0.0.0.0"
-  port: 8020 # NOTE: This port is used ONLY when running server.py directly. Gunicorn overrides this (see supervisord.conf).
-  reload: False # Default set to False - suitable for production
-  timeout_keep_alive: 300
-
-# Default LLM Configuration
-llm:
-  provider: "openai/gpt-4o-mini"
-  api_key_env: "OPENAI_API_KEY"
-  # api_key: sk-...  # If you pass the API key directly then api_key_env will be ignored
-
-# Redis Configuration (Used by internal Redis server managed by supervisord)
-redis:
-  host: "localhost"
-  port: 6379
-  db: 0
-  password: ""
-  # ... other redis options ...
-
-# Rate Limiting Configuration
-rate_limiting:
-  enabled: True
-  default_limit: "1000/minute"
-  trusted_proxies: []
-  storage_uri: "memory://"  # Use "redis://localhost:6379" if you need persistent/shared limits
-
-# Security Configuration
-security:
-  enabled: false # Master toggle for security features
-  jwt_enabled: false # Enable JWT authentication (requires security.enabled=true)
-  https_redirect: false # Force HTTPS (requires security.enabled=true)
-  trusted_hosts: ["*"] # Allowed hosts (use specific domains in production)
-  headers: # Security headers (applied if security.enabled=true)
-    x_content_type_options: "nosniff"
-    x_frame_options: "DENY"
-    content_security_policy: "default-src 'self'"
-    strict_transport_security: "max-age=63072000; includeSubDomains"
-
-# Crawler Configuration
-crawler:
-  memory_threshold_percent: 95.0
-  rate_limiter:
-    base_delay: [1.0, 2.0] # Min/max delay between requests in seconds for dispatcher
-  timeouts:
-    stream_init: 30.0  # Timeout for stream initialization
-    batch_process: 300.0 # Timeout for non-streaming /crawl processing
-
-# Logging Configuration
-logging:
-  level: "INFO"
-  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-
-# Observability Configuration
-observability:
-  prometheus:
-    enabled: True
-    endpoint: "/metrics"
-  health_check:
-    endpoint: "/health"
-```
-
-*(JWT Authentication section remains the same, just note the default port is now 11235 for requests)*
-
-*(Configuration Tips and Best Practices remain the same)*
-
-### Customizing Your Configuration
-
-You can override the default `config.yml`.
-
-#### Method 1: Modify Before Build
-
-1.  Edit the `deploy/docker/config.yml` file in your local repository clone.
-2.  Build the image using `docker buildx` or `docker compose --profile local-... up --build`. The modified file will be copied into the image.
-
-#### Method 2: Runtime Mount (Recommended for Custom Deploys)
-
-1.  Create your custom configuration file, e.g., `my-custom-config.yml` locally. Ensure it contains all necessary sections.
-2.  Mount it when running the container:
-
-    *   **Using `docker run`:**
-        ```bash
-        # Assumes my-custom-config.yml is in the current directory
-        docker run -d -p 11235:11235 \
-          --name crawl4ai-custom-config \
-          --env-file .llm.env \
-          --shm-size=1g \
-          -v $(pwd)/my-custom-config.yml:/app/config.yml \
-          unclecode/crawl4ai:latest # Or your specific tag
-        ```
-
-    *   **Using `docker-compose.yml`:** Add a `volumes` section to the service definition:
-        ```yaml
-        services:
-          crawl4ai-hub-amd64: # Or your chosen service
-            image: unclecode/crawl4ai:latest
-            profiles: ["hub-amd64"]
-            <<: *base-config
-            volumes:
-              # Mount local custom config over the default one in the container
-              - ./my-custom-config.yml:/app/config.yml
-              # Keep the shared memory volume from base-config
-              - /dev/shm:/dev/shm
-        ```
-        *(Note: Ensure `my-custom-config.yml` is in the same directory as `docker-compose.yml`)*
-
-> 💡 When mounting, your custom file *completely replaces* the default one. Ensure it's a valid and complete configuration.
-
-### Configuration Recommendations
-
-1. **Security First** 🔒
-   - Always enable security in production
-   - Use specific trusted_hosts instead of wildcards
-   - Set up proper rate limiting to protect your server
-   - Consider your environment before enabling HTTPS redirect
-
-2. **Resource Management** 💻
-   - Adjust memory_threshold_percent based on available RAM
-   - Set timeouts according to your content size and network conditions
-   - Use Redis for rate limiting in multi-container setups
-
-3. **Monitoring** 📊
-   - Enable Prometheus if you need metrics
-   - Set DEBUG logging in development, INFO in production
-   - Regular health check monitoring is crucial
-
-4. **Performance Tuning** ⚡
-   - Start with conservative rate limiter delays
-   - Increase batch_process timeout for large content
-   - Adjust stream_init timeout based on initial response times
-
-## Getting Help
-
-We're here to help you succeed with Crawl4AI! Here's how to get support:
-
-- 📖 Check our [full documentation](https://docs.crawl4ai.com)
-- 🐛 Found a bug? [Open an issue](https://github.com/unclecode/crawl4ai/issues)
-- 💬 Join our [Discord community](https://discord.gg/crawl4ai)
-- ⭐ Star us on GitHub to show support!
-
-## Summary
-
-In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment:
-- Building and running the Docker container
-- Configuring the environment
-- Making API requests with proper typing
-- Using the Python SDK
-- Monitoring your deployment
-
-Remember, the examples in the `examples` folder are your friends - they show real-world usage patterns that you can adapt for your needs.
-
-Keep exploring, and don't hesitate to reach out if you need help! We're building something amazing together. 🚀
-
-Happy crawling! 🕷️

deploy/docker/api.py

@@ -388,25 +388,21 @@ async def handle_crawl_request(
             )
         )
 
-        crawler: AsyncWebCrawler = AsyncWebCrawler(config=browser_config)
-        await crawler.start()
-        results = []
-        func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
-        partial_func = partial(func, 
-                                urls[0] if len(urls) == 1 else urls, 
-                                config=crawler_config, 
-                                dispatcher=dispatcher)
-        results = await partial_func()
-        await crawler.close()
-        return {
-            "success": True,
-            "results": [result.model_dump() for result in results]
-        }
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            results = []
+            func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
+            partial_func = partial(func, 
+                                   urls[0] if len(urls) == 1 else urls, 
+                                   config=crawler_config, 
+                                   dispatcher=dispatcher)
+            results = await partial_func()
+            return {
+                "success": True,
+                "results": [result.model_dump() for result in results]
+            }
 
     except Exception as e:
         logger.error(f"Crawl error: {str(e)}", exc_info=True)
-        if 'crawler' in locals():
-            await crawler.close()
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail=str(e)

deploy/docker/config.yml

@@ -4,7 +4,7 @@ app:
   version: "1.0.0"
   host: "0.0.0.0"
   port: 8020
-  reload: False
+  reload: True
   timeout_keep_alive: 300
 
 # Default LLM Configuration
@@ -68,4 +68,4 @@ observability:
     enabled: True
     endpoint: "/metrics"
   health_check:
-    endpoint: "/health"
+    endpoint: "/health"

deploy/docker/requirements.txt

@@ -1,3 +1,4 @@
+crawl4ai
 fastapi
 uvicorn
 gunicorn>=23.0.0

deploy/docker/supervisord.conf

@@ -1,28 +1,12 @@
 [supervisord]
-nodaemon=true                   ; Run supervisord in the foreground
-logfile=/dev/null               ; Log supervisord output to stdout/stderr
-logfile_maxbytes=0
+nodaemon=true
 
 [program:redis]
-command=/usr/bin/redis-server --loglevel notice ; Path to redis-server on Alpine
-user=appuser                    ; Run redis as our non-root user
+command=redis-server
 autorestart=true
 priority=10
-stdout_logfile=/dev/stdout      ; Redirect redis stdout to container stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr      ; Redirect redis stderr to container stderr
-stderr_logfile_maxbytes=0
 
 [program:gunicorn]
-command=/usr/local/bin/gunicorn --bind 0.0.0.0:11235 --workers 2 --threads 2 --timeout 120 --graceful-timeout 30 --keep-alive 60 --log-level info --worker-class uvicorn.workers.UvicornWorker server:app
-directory=/app                  ; Working directory for the app
-user=appuser                    ; Run gunicorn as our non-root user
+command=gunicorn --bind 0.0.0.0:8000 --workers 4 --threads 2 --timeout 300 --graceful-timeout 60 --keep-alive 65 --log-level debug --worker-class uvicorn.workers.UvicornWorker --max-requests 1000 --max-requests-jitter 50 server:app
 autorestart=true
-priority=20
-environment=PYTHONUNBUFFERED=1  ; Ensure Python output is sent straight to logs
-stdout_logfile=/dev/stdout      ; Redirect gunicorn stdout to container stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr      ; Redirect gunicorn stderr to container stderr
-stderr_logfile_maxbytes=0
-
-# Optional: Add filebeat or other logging agents here if needed
+priority=20

docker-compose.yml

@@ -1,30 +1,15 @@
-# docker-compose.yml
-
-# Base configuration anchor for reusability
+# Base configuration (not a service, just a reusable config block)
 x-base-config: &base-config
   ports:
-    # Map host port 11235 to container port 11235 (where Gunicorn will listen)
     - "11235:11235"
-    # - "8080:8080" # Uncomment if needed
-
-  # Load API keys primarily from .llm.env file
-  # Create .llm.env in the root directory .llm.env.example
-  env_file:
-    - .llm.env
-
-  # Define environment variables, allowing overrides from host environment
-  # Syntax ${VAR:-} uses host env var 'VAR' if set, otherwise uses value from .llm.env
+    - "8000:8000"
+    - "9222:9222"
+    - "8080:8080"
   environment:
+    - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
     - OPENAI_API_KEY=${OPENAI_API_KEY:-}
-    - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
-    - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
-    - GROQ_API_KEY=${GROQ_API_KEY:-}
-    - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
-    - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
-    - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-}
-
+    - CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
   volumes:
-    # Mount /dev/shm for Chromium/Playwright performance
     - /dev/shm:/dev/shm
   deploy:
     resources:
@@ -34,47 +19,47 @@ x-base-config: &base-config
         memory: 1G
   restart: unless-stopped
   healthcheck:
-    # IMPORTANT: Ensure Gunicorn binds to 11235 in supervisord.conf
     test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
     interval: 30s
     timeout: 10s
     retries: 3
-    start_period: 40s # Give the server time to start
-  # Run the container as the non-root user defined in the Dockerfile
-  user: "appuser"
+    start_period: 40s
 
 services:
-  # --- Local Build Services ---
-  crawl4ai-local-amd64:
+  # Local build services for different platforms
+  crawl4ai-amd64:
     build:
-      context: . # Build context is the root directory
-      dockerfile: Dockerfile # Dockerfile is in the root directory
+      context: .
+      dockerfile: Dockerfile
       args:
-        INSTALL_TYPE: ${INSTALL_TYPE:-default}
-        ENABLE_GPU: ${ENABLE_GPU:-false}
-        # PYTHON_VERSION arg is omitted as it's fixed by 'FROM python:3.10-slim' in Dockerfile
-    platform: linux/amd64
+        PYTHON_VERSION: "3.10"
+        INSTALL_TYPE: ${INSTALL_TYPE:-basic}
+        ENABLE_GPU: false
+      platforms:
+        - linux/amd64
     profiles: ["local-amd64"]
-    <<: *base-config # Inherit base configuration
+    <<: *base-config  # extends yerine doğrudan yapılandırmayı dahil ettik
 
-  crawl4ai-local-arm64:
+  crawl4ai-arm64:
     build:
-      context: . # Build context is the root directory
-      dockerfile: Dockerfile # Dockerfile is in the root directory
+      context: .
+      dockerfile: Dockerfile
       args:
-        INSTALL_TYPE: ${INSTALL_TYPE:-default}
-        ENABLE_GPU: ${ENABLE_GPU:-false}
-    platform: linux/arm64
+        PYTHON_VERSION: "3.10"
+        INSTALL_TYPE: ${INSTALL_TYPE:-basic}
+        ENABLE_GPU: false
+      platforms:
+        - linux/arm64
     profiles: ["local-arm64"]
     <<: *base-config
 
-  # --- Docker Hub Image Services ---
+  # Hub services for different platforms and versions
   crawl4ai-hub-amd64:
-    image: unclecode/crawl4ai:${VERSION:-latest}-amd64
+    image: unclecode/crawl4ai:${VERSION:-basic}-amd64
     profiles: ["hub-amd64"]
     <<: *base-config
 
   crawl4ai-hub-arm64:
-    image: unclecode/crawl4ai:${VERSION:-latest}-arm64
+    image: unclecode/crawl4ai:${VERSION:-basic}-arm64
     profiles: ["hub-arm64"]
     <<: *base-config

docs/examples/crypto_analysis_example.py

@@ -18,20 +18,11 @@ Key Features:
 
 import asyncio
 import pandas as pd
-import numpy as np
-import re
 import plotly.express as px
-from crawl4ai import (
-    AsyncWebCrawler,
-    BrowserConfig,
-    CrawlerRunConfig,
-    CacheMode,
-    LXMLWebScrapingStrategy,
-)
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LXMLWebScrapingStrategy
 from crawl4ai import CrawlResult
 from typing import List
-
-__current_dir__ = __file__.rsplit("/", 1)[0]
+from IPython.display import HTML
 
 class CryptoAlphaGenerator:
     """
@@ -40,319 +31,134 @@ class CryptoAlphaGenerator:
     - Liquidity scores
     - Momentum-risk ratios
     - Machine learning-inspired trading signals
-
+    
     Methods:
     analyze_tables(): Process raw tables into trading insights
     create_visuals(): Generate institutional-grade visualizations
     generate_insights(): Create plain English trading recommendations
     """
-
+    
     def clean_data(self, df: pd.DataFrame) -> pd.DataFrame:
         """
-        Convert crypto market data to machine-readable format.
-        Handles currency symbols, units (B=Billions), and percentage values.
+        Convert crypto market data to machine-readable format
+        Handles currency symbols, units (B=Billions), and percentage values
         """
-        # Make a copy to avoid SettingWithCopyWarning
-        df = df.copy()
-        
-        # Clean Price column (handle currency symbols)
-        df["Price"] = df["Price"].astype(str).str.replace("[^\d.]", "", regex=True).astype(float)
-        
-        # Handle Market Cap and Volume, considering both Billions and Trillions
-        def convert_large_numbers(value):
-            if pd.isna(value):
-                return float('nan')
-            value = str(value)
-            multiplier = 1
-            if 'B' in value:
-                multiplier = 1e9
-            elif 'T' in value:
-                multiplier = 1e12
-            # Handle cases where the value might already be numeric
-            cleaned_value = re.sub(r"[^\d.]", "", value)
-            return float(cleaned_value) * multiplier if cleaned_value else float('nan')
-        
-        df["Market Cap"] = df["Market Cap"].apply(convert_large_numbers)
-        df["Volume(24h)"] = df["Volume(24h)"].apply(convert_large_numbers)
+        # Clean numeric columns
+        df['Price'] = df['Price'].str.replace('[^\d.]', '', regex=True).astype(float)
+        df['Market Cap'] = df['Market Cap'].str.extract(r'\$([\d.]+)B')[0].astype(float) * 1e9
+        df['Volume(24h)'] = df['Volume(24h)'].str.extract(r'\$([\d.]+)B')[0].astype(float) * 1e9
         
         # Convert percentages to decimal values
-        for col in ["1h %", "24h %", "7d %"]:
-            if col in df.columns:
-                # First ensure it's string, then clean
-                df[col] = (
-                    df[col].astype(str)
-                    .str.replace("%", "")
-                    .str.replace(",", ".")
-                    .replace("nan", np.nan)
-                )
-                df[col] = pd.to_numeric(df[col], errors='coerce') / 100
-        
+        for col in ['1h %', '24h %', '7d %']:
+            df[col] = df[col].str.replace('%', '').astype(float) / 100
+            
         return df
 
     def calculate_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
         """
         Compute advanced trading metrics used by quantitative funds:
-
+        
         1. Volume/Market Cap Ratio - Measures liquidity efficiency
-           (High ratio = Underestimated attention, and small-cap = higher growth potential)
-
-        2. Volatility Score - Risk-adjusted momentum potential - Shows how stable is the trend
+           (High ratio = Underestimated attention)
+           
+        2. Volatility Score - Risk-adjusted momentum potential
            (STD of 1h/24h/7d returns)
-
-        3. Momentum Score - Weighted average of returns - Shows how strong is the trend
+           
+        3. Momentum Score - Weighted average of returns
            (1h:30% + 24h:50% + 7d:20%)
-
+           
         4. Volume Anomaly - 3σ deviation detection
-           (Flags potential insider activity) - Unusual trading activity – Flags coins with volume spikes (potential insider buying or news).
+           (Flags potential insider activity)
         """
         # Liquidity Metrics
-        df["Volume/Market Cap Ratio"] = df["Volume(24h)"] / df["Market Cap"]
-
+        df['Volume/Market Cap Ratio'] = df['Volume(24h)'] / df['Market Cap']
+        
         # Risk Metrics
-        df["Volatility Score"] = df[["1h %", "24h %", "7d %"]].std(axis=1)
-
+        df['Volatility Score'] = df[['1h %','24h %','7d %']].std(axis=1)
+        
         # Momentum Metrics
-        df["Momentum Score"] = df["1h %"] * 0.3 + df["24h %"] * 0.5 + df["7d %"] * 0.2
-
+        df['Momentum Score'] = (df['1h %']*0.3 + df['24h %']*0.5 + df['7d %']*0.2)
+        
         # Anomaly Detection
-        median_vol = df["Volume(24h)"].median()
-        df["Volume Anomaly"] = df["Volume(24h)"] > 3 * median_vol
-
+        median_vol = df['Volume(24h)'].median()
+        df['Volume Anomaly'] = df['Volume(24h)'] > 3 * median_vol
+        
         # Value Flags
-        # Undervalued Flag - Low market cap and high momentum
-        # (High growth potential and low attention)
-        df["Undervalued Flag"] = (df["Market Cap"] < 1e9) & (
-            df["Momentum Score"] > 0.05
-        )
-        # Liquid Giant Flag - High volume/market cap ratio and large market cap
-        # (High liquidity and large market cap = institutional interest)
-        df["Liquid Giant"] = (df["Volume/Market Cap Ratio"] > 0.15) & (
-            df["Market Cap"] > 1e9
-        )
-
+        df['Undervalued Flag'] = (df['Market Cap'] < 1e9) & (df['Momentum Score'] > 0.05)
+        df['Liquid Giant'] = (df['Volume/Market Cap Ratio'] > 0.15) & (df['Market Cap'] > 1e9)
+        
         return df
 
-    def generate_insights_simple(self, df: pd.DataFrame) -> str:
+    def create_visuals(self, df: pd.DataFrame) -> dict:
         """
-        Generates an ultra-actionable crypto trading report with:
-        - Risk-tiered opportunities (High/Medium/Low)
-        - Concrete examples for each trade type
-        - Entry/exit strategies spelled out
-        - Visual cues for quick scanning
+        Generate three institutional-grade visualizations:
+        
+        1. 3D Market Map - X:Size, Y:Liquidity, Z:Momentum
+        2. Liquidity Tree - Color:Volume Efficiency
+        3. Momentum Leaderboard - Top sustainable movers
         """
-        report = [
-            "🚀 **CRYPTO TRADING CHEAT SHEET** 🚀",
-            "*Based on quantitative signals + hedge fund tactics*",
-            "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-        ]
-
-        # 1. HIGH-RISK: Undervalued Small-Caps (Momentum Plays)
-        high_risk = df[df["Undervalued Flag"]].sort_values("Momentum Score", ascending=False)
-        if not high_risk.empty:
-            example_coin = high_risk.iloc[0]
-            report.extend([
-                "\n🔥 **HIGH-RISK: Rocket Fuel Small-Caps**",
-                f"*Example Trade:* {example_coin['Name']} (Price: ${example_coin['Price']:.6f})",
-                "📊 *Why?* Tiny market cap (<$1B) but STRONG momentum (+{:.0f}% last week)".format(example_coin['7d %']*100),
-                "🎯 *Strategy:*",
-                "1. Wait for 5-10% dip from recent high (${:.6f} → Buy under ${:.6f})".format(
-                    example_coin['Price'] / (1 - example_coin['24h %']),  # Approx recent high
-                    example_coin['Price'] * 0.95
-                ),
-                "2. Set stop-loss at -10% (${:.6f})".format(example_coin['Price'] * 0.90),
-                "3. Take profit at +20% (${:.6f})".format(example_coin['Price'] * 1.20),
-                "⚠️ *Risk Warning:* These can drop 30% fast! Never bet more than 5% of your portfolio.",
-                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-            ])
-
-        # 2. MEDIUM-RISK: Liquid Giants (Swing Trades)
-        medium_risk = df[df["Liquid Giant"]].sort_values("Volume/Market Cap Ratio", ascending=False)
-        if not medium_risk.empty:
-            example_coin = medium_risk.iloc[0]
-            report.extend([
-                "\n💎 **MEDIUM-RISK: Liquid Giants (Safe Swing Trades)**",
-                f"*Example Trade:* {example_coin['Name']} (Market Cap: ${example_coin['Market Cap']/1e9:.1f}B)",
-                "📊 *Why?* Huge volume (${:.1f}M/day) makes it easy to enter/exit".format(example_coin['Volume(24h)']/1e6),
-                "🎯 *Strategy:*",
-                "1. Buy when 24h volume > 15% of market cap (Current: {:.0f}%)".format(example_coin['Volume/Market Cap Ratio']*100),
-                "2. Hold 1-4 weeks (Big coins trend longer)",
-                "3. Exit when momentum drops below 5% (Current: {:.0f}%)".format(example_coin['Momentum Score']*100),
-                "📉 *Pro Tip:* Watch Bitcoin's trend - if BTC drops 5%, these usually follow.",
-                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-            ])
-
-        # 3. LOW-RISK: Stable Momentum (DCA Targets)
-        low_risk = df[
-            (df["Momentum Score"] > 0.05) & 
-            (df["Volatility Score"] < 0.03)
-        ].sort_values("Market Cap", ascending=False)
-        if not low_risk.empty:
-            example_coin = low_risk.iloc[0]
-            report.extend([
-                "\n🛡️ **LOW-RISK: Steady Climbers (DCA & Forget)**",
-                f"*Example Trade:* {example_coin['Name']} (Volatility: {example_coin['Volatility Score']:.2f}/5)",
-                "📊 *Why?* Rises steadily (+{:.0f}%/week) with LOW drama".format(example_coin['7d %']*100),
-                "🎯 *Strategy:*",
-                "1. Buy small amounts every Tuesday/Friday (DCA)",
-                "2. Hold for 3+ months (Compound gains work best here)",
-                "3. Sell 10% at every +25% milestone",
-                "💰 *Best For:* Long-term investors who hate sleepless nights",
-                "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-            ])
-
-        # Volume Spike Alerts
-        anomalies = df[df["Volume Anomaly"]].sort_values("Volume(24h)", ascending=False)
-        if not anomalies.empty:
-            example_coin = anomalies.iloc[0]
-            report.extend([
-                "\n🚨 **Volume Spike Alert (Possible News/Whale Action)**",
-                f"*Coin:* {example_coin['Name']} (Volume: ${example_coin['Volume(24h)']/1e6:.1f}M, usual: ${example_coin['Volume(24h)']/3/1e6:.1f}M)",
-                "🔍 *Check:* Twitter/CoinGecko for news before trading",
-                "⚡ *If no news:* Could be insider buying - watch price action:",
-                "- Break above today's high → Buy with tight stop-loss",
-                "- Fade back down → Avoid (may be a fakeout)"
-            ])
-
-        # Pro Tip Footer
-        report.append("\n✨ *Pro Tip:* Bookmark this report & check back in 24h to see if signals held up.")
-
-        return "\n".join(report)
+        # 3D Market Overview
+        fig1 = px.scatter_3d(
+            df, 
+            x='Market Cap', 
+            y='Volume/Market Cap Ratio',
+            z='Momentum Score',
+            size='Volatility Score',
+            color='Volume Anomaly',
+            hover_name='Name',
+            title='Smart Money Market Map: Spot Overlooked Opportunities',
+            labels={'Market Cap': 'Size (Log $)', 'Volume/Market Cap Ratio': 'Liquidity Power'},
+            log_x=True,
+            template='plotly_dark'
+        )
+        
+        # Liquidity Efficiency Tree
+        fig2 = px.treemap(
+            df,
+            path=['Name'], 
+            values='Market Cap',
+            color='Volume/Market Cap Ratio',
+            hover_data=['Momentum Score'],
+            title='Liquidity Forest: Green = High Trading Efficiency',
+            color_continuous_scale='RdYlGn'
+        )
+        
+        # Momentum Leaders
+        fig3 = px.bar(
+            df.sort_values('Momentum Score', ascending=False).head(10),
+            x='Name', 
+            y='Momentum Score',
+            color='Volatility Score',
+            title='Sustainable Momentum Leaders (Low Volatility + High Growth)',
+            text='7d %',
+            template='plotly_dark'
+        )
+        
+        return {'market_map': fig1, 'liquidity_tree': fig2, 'momentum_leaders': fig3}
 
     def generate_insights(self, df: pd.DataFrame) -> str:
         """
-        Generates a tactical trading report with:
-        - Top 3 trades per risk level (High/Medium/Low)
-        - Auto-calculated entry/exit prices
-        - BTC chart toggle tip
+        Create plain English trading insights explaining:
+        - Volume spikes and their implications
+        - Risk-reward ratios of top movers
+        - Liquidity warnings for large positions
         """
-        # Filter top candidates for each risk level
-        high_risk = (
-            df[df["Undervalued Flag"]]
-            .sort_values("Momentum Score", ascending=False)
-            .head(3)
-        )
-        medium_risk = (
-            df[df["Liquid Giant"]]
-            .sort_values("Volume/Market Cap Ratio", ascending=False)
-            .head(3)
-        )
-        low_risk = (
-            df[(df["Momentum Score"] > 0.05) & (df["Volatility Score"] < 0.03)]
-            .sort_values("Momentum Score", ascending=False)
-            .head(3)
-        )
-
-        report = ["# 🎯 Crypto Trading Tactical Report (Top 3 Per Risk Tier)"]
+        top_coin = df.sort_values('Momentum Score', ascending=False).iloc[0]
+        anomaly_coins = df[df['Volume Anomaly']].sort_values('Volume(24h)', ascending=False)
         
-        # 1. High-Risk Trades (Small-Cap Momentum)
-        if not high_risk.empty:
-            report.append("\n## 🔥 HIGH RISK: Small-Cap Rockets (5-50% Potential)")
-            for i, coin in high_risk.iterrows():
-                current_price = coin["Price"]
-                entry = current_price * 0.95  # -5% dip
-                stop_loss = current_price * 0.90  # -10%
-                take_profit = current_price * 1.20  # +20%
-                
-                report.append(
-                    f"\n### {coin['Name']} (Momentum: {coin['Momentum Score']:.1%})"
-                    f"\n- **Current Price:** ${current_price:.4f}"
-                    f"\n- **Entry:** < ${entry:.4f} (Wait for pullback)"
-                    f"\n- **Stop-Loss:** ${stop_loss:.4f} (-10%)"
-                    f"\n- **Target:** ${take_profit:.4f} (+20%)"
-                    f"\n- **Risk/Reward:** 1:2"
-                    f"\n- **Watch:** Volume spikes above {coin['Volume(24h)']/1e6:.1f}M"
-                )
-
-        # 2. Medium-Risk Trades (Liquid Giants)
-        if not medium_risk.empty:
-            report.append("\n## 💎 MEDIUM RISK: Liquid Swing Trades (10-30% Potential)")
-            for i, coin in medium_risk.iterrows():
-                current_price = coin["Price"]
-                entry = current_price * 0.98  # -2% dip
-                stop_loss = current_price * 0.94  # -6%
-                take_profit = current_price * 1.15  # +15%
-                
-                report.append(
-                    f"\n### {coin['Name']} (Liquidity Score: {coin['Volume/Market Cap Ratio']:.1%})"
-                    f"\n- **Current Price:** ${current_price:.2f}"
-                    f"\n- **Entry:** < ${entry:.2f} (Buy slight dips)"
-                    f"\n- **Stop-Loss:** ${stop_loss:.2f} (-6%)"
-                    f"\n- **Target:** ${take_profit:.2f} (+15%)"
-                    f"\n- **Hold Time:** 1-3 weeks"
-                    f"\n- **Key Metric:** Volume/Cap > 15%"
-                )
-
-        # 3. Low-Risk Trades (Stable Momentum)
-        if not low_risk.empty:
-            report.append("\n## 🛡️ LOW RISK: Steady Gainers (5-15% Potential)")
-            for i, coin in low_risk.iterrows():
-                current_price = coin["Price"]
-                entry = current_price * 0.99  # -1% dip
-                stop_loss = current_price * 0.97  # -3%
-                take_profit = current_price * 1.10  # +10%
-                
-                report.append(
-                    f"\n### {coin['Name']} (Stability Score: {1/coin['Volatility Score']:.1f}x)"
-                    f"\n- **Current Price:** ${current_price:.2f}"
-                    f"\n- **Entry:** < ${entry:.2f} (Safe zone)"
-                    f"\n- **Stop-Loss:** ${stop_loss:.2f} (-3%)"
-                    f"\n- **Target:** ${take_profit:.2f} (+10%)"
-                    f"\n- **DCA Suggestion:** 3 buys over 72 hours"
-                )
-
-        # Volume Anomaly Alert
-        anomalies = df[df["Volume Anomaly"]].sort_values("Volume(24h)", ascending=False).head(2)
-        if not anomalies.empty:
-            report.append("\n⚠️ **Volume Spike Alerts**")
-            for i, coin in anomalies.iterrows():
-                report.append(
-                    f"- {coin['Name']}: Volume {coin['Volume(24h)']/1e6:.1f}M "
-                    f"(3x normal) | Price moved: {coin['24h %']:.1%}"
-                )
-
-        # Pro Tip
-        report.append(
-            "\n📊 **Chart Hack:** Hide BTC in visuals:\n"
-            "```python\n"
-            "# For 3D Map:\n"
-            "fig.update_traces(visible=False, selector={'name':'Bitcoin'})\n"
-            "# For Treemap:\n"
-            "df = df[df['Name'] != 'Bitcoin']\n"
-            "```"
-        )
-
-        return "\n".join(report)
-
-    def create_visuals(self, df: pd.DataFrame) -> dict:
-        """Enhanced visuals with BTC toggle support"""
-        # 3D Market Map (with BTC toggle hint)
-        fig1 = px.scatter_3d(
-            df,
-            x="Market Cap",
-            y="Volume/Market Cap Ratio",
-            z="Momentum Score",
-            color="Name",  # Color by name to allow toggling
-            hover_name="Name",
-            title="Market Map (Toggle BTC in legend to focus on alts)",
-            log_x=True
-        )
-        fig1.update_traces(
-            marker=dict(size=df["Volatility Score"]*100 + 5)  # Dynamic sizing
-        )
+        report = f"""
+        🚀 Top Alpha Opportunity: {top_coin['Name']}
+        - Momentum Score: {top_coin['Momentum Score']:.2%} (Top 1%)
+        - Risk-Reward Ratio: {top_coin['Momentum Score']/top_coin['Volatility Score']:.1f}
+        - Liquidity Warning: {'✅ Safe' if top_coin['Liquid Giant'] else '⚠️ Thin Markets'}
         
-        # Liquidity Tree (exclude BTC if too dominant)
-        if df[df["Name"] == "BitcoinBTC"]["Market Cap"].values[0] > df["Market Cap"].median() * 10:
-            df = df[df["Name"] != "BitcoinBTC"]
+        🔥 Volume Spikes Detected ({len(anomaly_coins)} coins):
+        {anomaly_coins[['Name', 'Volume(24h)']].head(3).to_markdown(index=False)}
         
-        fig2 = px.treemap(
-            df,
-            path=["Name"],
-            values="Market Cap",
-            color="Volume/Market Cap Ratio",
-            title="Liquidity Tree (BTC auto-removed if dominant)"
-        )
-        
-        return {"market_map": fig1, "liquidity_tree": fig2}
+        💡 Smart Money Tip: Coins with Volume/Cap > 15% and Momentum > 5% 
+        historically outperform by 22% weekly returns.
+        """
+        return report
 
 async def main():
     """
@@ -365,79 +171,60 @@ async def main():
     """
     # Configure browser with anti-detection features
     browser_config = BrowserConfig(
-        headless=False,
+        headless=True,
+        stealth=True,
+        block_resources=["image", "media"]
     )
-
+    
     # Initialize crawler with smart table detection
     crawler = AsyncWebCrawler(config=browser_config)
     await crawler.start()
-
+    
     try:
         # Set up scraping parameters
         crawl_config = CrawlerRunConfig(
             cache_mode=CacheMode.BYPASS,
-            table_score_threshold=8,  # Strict table detection
-            keep_data_attributes=True,
-            scraping_strategy=LXMLWebScrapingStrategy(),
-            scan_full_page=True,
-            scroll_delay=0.2,
+            scraping_strategy=LXMLWebScrapingStrategy(
+                table_score_threshold=8,  # Strict table detection
+                keep_data_attributes=True
+            )
         )
-
-        # # Execute market data extraction
-        # results: List[CrawlResult] = await crawler.arun(
-        #     url="https://coinmarketcap.com/?page=1", config=crawl_config
-        # )
-
-        # # Process results
-        # raw_df = pd.DataFrame()
-        # for result in results:
-        #     if result.success and result.media["tables"]:
-        #         # Extract primary market table
-        #         # DataFrame
-        #         raw_df = pd.DataFrame(
-        #             result.media["tables"][0]["rows"],
-        #             columns=result.media["tables"][0]["headers"],
-        #         )
-        #         break
-
-
-        # This is for debugging only
-        # ////// Remove this in production from here..
-        # Save raw data for debugging
-        # raw_df.to_csv(f"{__current_dir__}/tmp/raw_crypto_data.csv", index=False)
-        # print("🔍 Raw data saved to 'raw_crypto_data.csv'")
-
-        # Read from file for debugging
-        raw_df = pd.read_csv(f"{__current_dir__}/tmp/raw_crypto_data.csv")
-        # ////// ..to here
-
-        # Select top 20
-        raw_df = raw_df.head(50)
-        # Remove "Buy" from name
-        raw_df["Name"] = raw_df["Name"].str.replace("Buy", "")
-
-        # Initialize analysis engine
-        analyzer = CryptoAlphaGenerator()
-        clean_df = analyzer.clean_data(raw_df)
-        analyzed_df = analyzer.calculate_metrics(clean_df)
-
-        # Generate outputs
-        visuals = analyzer.create_visuals(analyzed_df)
-        insights = analyzer.generate_insights(analyzed_df)
-
-        # Save visualizations
-        visuals["market_map"].write_html(f"{__current_dir__}/tmp/market_map.html")
-        visuals["liquidity_tree"].write_html(f"{__current_dir__}/tmp/liquidity_tree.html")
-
-        # Display results
-        print("🔑 Key Trading Insights:")
-        print(insights)
-        print("\n📊 Open 'market_map.html' for interactive analysis")
-        print("\n📊 Open 'liquidity_tree.html' for interactive analysis")
+        
+        # Execute market data extraction
+        results: List[CrawlResult] = await crawler.arun(
+            url='https://coinmarketcap.com/?page=1',
+            config=crawl_config
+        )
+        
+        # Process results
+        for result in results:
+            if result.success and result.media['tables']:
+                # Extract primary market table
+                raw_df = pd.DataFrame(
+                    result.media['tables'][0]['rows'],
+                    columns=result.media['tables'][0]['headers']
+                )
+                
+                # Initialize analysis engine
+                analyzer = CryptoAlphaGenerator()
+                clean_df = analyzer.clean_data(raw_df)
+                analyzed_df = analyzer.calculate_metrics(clean_df)
+                
+                # Generate outputs
+                visuals = analyzer.create_visuals(analyzed_df)
+                insights = analyzer.generate_insights(analyzed_df)
+                
+                # Save visualizations
+                visuals['market_map'].write_html("market_map.html")
+                visuals['liquidity_tree'].write_html("liquidity_tree.html")
+                
+                # Display results
+                print("🔑 Key Trading Insights:")
+                print(insights)
+                print("\n📊 Open 'market_map.html' for interactive analysis")
 
     finally:
         await crawler.close()
 
-
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(main())

docs/examples/network_console_capture_example.py

@@ -1,477 +0,0 @@
-import asyncio
-import json
-import os
-import base64
-from pathlib import Path
-from typing import List, Dict, Any
-from datetime import datetime
-
-from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, CrawlResult
-from crawl4ai import BrowserConfig
-
-__cur_dir__ = Path(__file__).parent
-
-# Create temp directory if it doesn't exist
-os.makedirs(os.path.join(__cur_dir__, "tmp"), exist_ok=True)
-
-async def demo_basic_network_capture():
-    """Basic network request capturing example"""
-    print("\n=== 1. Basic Network Request Capturing ===")
-    
-    async with AsyncWebCrawler() as crawler:
-        config = CrawlerRunConfig(
-            capture_network_requests=True,
-            wait_until="networkidle"  # Wait for network to be idle
-        )
-        
-        result = await crawler.arun(
-            url="https://example.com/",
-            config=config
-        )
-        
-        if result.success and result.network_requests:
-            print(f"Captured {len(result.network_requests)} network events")
-            
-            # Count by event type
-            event_types = {}
-            for req in result.network_requests:
-                event_type = req.get("event_type", "unknown")
-                event_types[event_type] = event_types.get(event_type, 0) + 1
-            
-            print("Event types:")
-            for event_type, count in event_types.items():
-                print(f"  - {event_type}: {count}")
-            
-            # Show a sample request and response
-            request = next((r for r in result.network_requests if r.get("event_type") == "request"), None)
-            response = next((r for r in result.network_requests if r.get("event_type") == "response"), None)
-            
-            if request:
-                print("\nSample request:")
-                print(f"  URL: {request.get('url')}")
-                print(f"  Method: {request.get('method')}")
-                print(f"  Headers: {list(request.get('headers', {}).keys())}")
-            
-            if response:
-                print("\nSample response:")
-                print(f"  URL: {response.get('url')}")
-                print(f"  Status: {response.get('status')} {response.get('status_text', '')}")
-                print(f"  Headers: {list(response.get('headers', {}).keys())}")
-
-async def demo_basic_console_capture():
-    """Basic console message capturing example"""
-    print("\n=== 2. Basic Console Message Capturing ===")
-    
-    # Create a simple HTML file with console messages
-    html_file = os.path.join(__cur_dir__, "tmp", "console_test.html")
-    with open(html_file, "w") as f:
-        f.write("""
-        <!DOCTYPE html>
-        <html>
-        <head>
-            <title>Console Test</title>
-        </head>
-        <body>
-            <h1>Console Message Test</h1>
-            <script>
-                console.log("This is a basic log message");
-                console.info("This is an info message");
-                console.warn("This is a warning message");
-                console.error("This is an error message");
-                
-                // Generate an error
-                try {
-                    nonExistentFunction();
-                } catch (e) {
-                    console.error("Caught error:", e);
-                }
-            </script>
-        </body>
-        </html>
-        """)
-    
-    async with AsyncWebCrawler() as crawler:
-        config = CrawlerRunConfig(
-            capture_console_messages=True,
-            wait_until="networkidle"  # Wait to make sure all scripts execute
-        )
-        
-        result = await crawler.arun(
-            url=f"file://{html_file}",
-            config=config
-        )
-        
-        if result.success and result.console_messages:
-            print(f"Captured {len(result.console_messages)} console messages")
-            
-            # Count by message type
-            message_types = {}
-            for msg in result.console_messages:
-                msg_type = msg.get("type", "unknown")
-                message_types[msg_type] = message_types.get(msg_type, 0) + 1
-            
-            print("Message types:")
-            for msg_type, count in message_types.items():
-                print(f"  - {msg_type}: {count}")
-            
-            # Show all messages
-            print("\nAll console messages:")
-            for i, msg in enumerate(result.console_messages, 1):
-                print(f"  {i}. [{msg.get('type', 'unknown')}] {msg.get('text', '')}")
-
-async def demo_combined_capture():
-    """Capturing both network requests and console messages"""
-    print("\n=== 3. Combined Network and Console Capture ===")
-    
-    async with AsyncWebCrawler() as crawler:
-        config = CrawlerRunConfig(
-            capture_network_requests=True,
-            capture_console_messages=True,
-            wait_until="networkidle"
-        )
-        
-        result = await crawler.arun(
-            url="https://httpbin.org/html",
-            config=config
-        )
-        
-        if result.success:
-            network_count = len(result.network_requests) if result.network_requests else 0
-            console_count = len(result.console_messages) if result.console_messages else 0
-            
-            print(f"Captured {network_count} network events and {console_count} console messages")
-            
-            # Save the captured data to a JSON file for analysis
-            output_file = os.path.join(__cur_dir__, "tmp", "capture_data.json")
-            with open(output_file, "w") as f:
-                json.dump({
-                    "url": result.url,
-                    "timestamp": datetime.now().isoformat(),
-                    "network_requests": result.network_requests,
-                    "console_messages": result.console_messages
-                }, f, indent=2)
-            
-            print(f"Full capture data saved to {output_file}")
-
-async def analyze_spa_network_traffic():
-    """Analyze network traffic of a Single-Page Application"""
-    print("\n=== 4. Analyzing SPA Network Traffic ===")
-    
-    async with AsyncWebCrawler(config=BrowserConfig(
-        headless=True,
-        viewport_width=1280,
-        viewport_height=800
-    )) as crawler:
-        config = CrawlerRunConfig(
-            capture_network_requests=True,
-            capture_console_messages=True,
-            # Wait longer to ensure all resources are loaded
-            wait_until="networkidle",
-            page_timeout=60000,  # 60 seconds
-        )
-        
-        result = await crawler.arun(
-            url="https://weather.com",
-            config=config
-        )
-        
-        if result.success and result.network_requests:
-            # Extract different types of requests
-            requests = []
-            responses = []
-            failures = []
-            
-            for event in result.network_requests:
-                event_type = event.get("event_type")
-                if event_type == "request":
-                    requests.append(event)
-                elif event_type == "response":
-                    responses.append(event)
-                elif event_type == "request_failed":
-                    failures.append(event)
-            
-            print(f"Captured {len(requests)} requests, {len(responses)} responses, and {len(failures)} failures")
-            
-            # Analyze request types
-            resource_types = {}
-            for req in requests:
-                resource_type = req.get("resource_type", "unknown")
-                resource_types[resource_type] = resource_types.get(resource_type, 0) + 1
-            
-            print("\nResource types:")
-            for resource_type, count in sorted(resource_types.items(), key=lambda x: x[1], reverse=True):
-                print(f"  - {resource_type}: {count}")
-            
-            # Analyze API calls
-            api_calls = [r for r in requests if "api" in r.get("url", "").lower()]
-            if api_calls:
-                print(f"\nDetected {len(api_calls)} API calls:")
-                for i, call in enumerate(api_calls[:5], 1):  # Show first 5
-                    print(f"  {i}. {call.get('method')} {call.get('url')}")
-                if len(api_calls) > 5:
-                    print(f"     ... and {len(api_calls) - 5} more")
-            
-            # Analyze response status codes
-            status_codes = {}
-            for resp in responses:
-                status = resp.get("status", 0)
-                status_codes[status] = status_codes.get(status, 0) + 1
-            
-            print("\nResponse status codes:")
-            for status, count in sorted(status_codes.items()):
-                print(f"  - {status}: {count}")
-            
-            # Analyze failures
-            if failures:
-                print("\nFailed requests:")
-                for i, failure in enumerate(failures[:5], 1):  # Show first 5
-                    print(f"  {i}. {failure.get('url')} - {failure.get('failure_text')}")
-                if len(failures) > 5:
-                    print(f"     ... and {len(failures) - 5} more")
-            
-            # Check for console errors
-            if result.console_messages:
-                errors = [msg for msg in result.console_messages if msg.get("type") == "error"]
-                if errors:
-                    print(f"\nDetected {len(errors)} console errors:")
-                    for i, error in enumerate(errors[:3], 1):  # Show first 3
-                        print(f"  {i}. {error.get('text', '')[:100]}...")
-                    if len(errors) > 3:
-                        print(f"     ... and {len(errors) - 3} more")
-            
-            # Save analysis to file
-            output_file = os.path.join(__cur_dir__, "tmp", "weather_network_analysis.json")
-            with open(output_file, "w") as f:
-                json.dump({
-                    "url": result.url,
-                    "timestamp": datetime.now().isoformat(),
-                    "statistics": {
-                        "request_count": len(requests),
-                        "response_count": len(responses),
-                        "failure_count": len(failures),
-                        "resource_types": resource_types,
-                        "status_codes": {str(k): v for k, v in status_codes.items()},
-                        "api_call_count": len(api_calls),
-                        "console_error_count": len(errors) if result.console_messages else 0
-                    },
-                    "network_requests": result.network_requests,
-                    "console_messages": result.console_messages
-                }, f, indent=2)
-            
-            print(f"\nFull analysis saved to {output_file}")
-
-async def demo_security_analysis():
-    """Using network capture for security analysis"""
-    print("\n=== 5. Security Analysis with Network Capture ===")
-    
-    async with AsyncWebCrawler() as crawler:
-        config = CrawlerRunConfig(
-            capture_network_requests=True,
-            capture_console_messages=True,
-            wait_until="networkidle"
-        )
-        
-        # A site that makes multiple third-party requests
-        result = await crawler.arun(
-            url="https://www.nytimes.com/",
-            config=config
-        )
-        
-        if result.success and result.network_requests:
-            print(f"Captured {len(result.network_requests)} network events")
-            
-            # Extract all domains
-            domains = set()
-            for req in result.network_requests:
-                if req.get("event_type") == "request":
-                    url = req.get("url", "")
-                    try:
-                        from urllib.parse import urlparse
-                        domain = urlparse(url).netloc
-                        if domain:
-                            domains.add(domain)
-                    except:
-                        pass
-            
-            print(f"\nDetected requests to {len(domains)} unique domains:")
-            main_domain = urlparse(result.url).netloc
-            
-            # Separate first-party vs third-party domains
-            first_party = [d for d in domains if main_domain in d]
-            third_party = [d for d in domains if main_domain not in d]
-            
-            print(f"  - First-party domains: {len(first_party)}")
-            print(f"  - Third-party domains: {len(third_party)}")
-            
-            # Look for potential trackers/analytics
-            tracking_keywords = ["analytics", "tracker", "pixel", "tag", "stats", "metric", "collect", "beacon"]
-            potential_trackers = []
-            
-            for domain in third_party:
-                if any(keyword in domain.lower() for keyword in tracking_keywords):
-                    potential_trackers.append(domain)
-            
-            if potential_trackers:
-                print(f"\nPotential tracking/analytics domains ({len(potential_trackers)}):")
-                for i, domain in enumerate(sorted(potential_trackers)[:10], 1):
-                    print(f"  {i}. {domain}")
-                if len(potential_trackers) > 10:
-                    print(f"     ... and {len(potential_trackers) - 10} more")
-            
-            # Check for insecure (HTTP) requests
-            insecure_requests = [
-                req.get("url") for req in result.network_requests 
-                if req.get("event_type") == "request" and req.get("url", "").startswith("http://")
-            ]
-            
-            if insecure_requests:
-                print(f"\nWarning: Found {len(insecure_requests)} insecure (HTTP) requests:")
-                for i, url in enumerate(insecure_requests[:5], 1):
-                    print(f"  {i}. {url}")
-                if len(insecure_requests) > 5:
-                    print(f"     ... and {len(insecure_requests) - 5} more")
-            
-            # Save security analysis to file
-            output_file = os.path.join(__cur_dir__, "tmp", "security_analysis.json")
-            with open(output_file, "w") as f:
-                json.dump({
-                    "url": result.url,
-                    "main_domain": main_domain,
-                    "timestamp": datetime.now().isoformat(),
-                    "analysis": {
-                        "total_requests": len([r for r in result.network_requests if r.get("event_type") == "request"]),
-                        "unique_domains": len(domains),
-                        "first_party_domains": first_party,
-                        "third_party_domains": third_party,
-                        "potential_trackers": potential_trackers,
-                        "insecure_requests": insecure_requests
-                    }
-                }, f, indent=2)
-            
-            print(f"\nFull security analysis saved to {output_file}")
-
-async def demo_performance_analysis():
-    """Using network capture for performance analysis"""
-    print("\n=== 6. Performance Analysis with Network Capture ===")
-    
-    async with AsyncWebCrawler() as crawler:
-        config = CrawlerRunConfig(
-            capture_network_requests=True,
-            page_timeout=60 * 2 * 1000  # 120 seconds
-        )
-        
-        result = await crawler.arun(
-            url="https://www.cnn.com/",
-            config=config
-        )
-        
-        if result.success and result.network_requests:
-            # Filter only response events with timing information
-            responses_with_timing = [
-                r for r in result.network_requests 
-                if r.get("event_type") == "response" and r.get("request_timing")
-            ]
-            
-            if responses_with_timing:
-                print(f"Analyzing timing for {len(responses_with_timing)} network responses")
-                
-                # Group by resource type
-                resource_timings = {}
-                for resp in responses_with_timing:
-                    url = resp.get("url", "")
-                    timing = resp.get("request_timing", {})
-                    
-                    # Determine resource type from URL extension
-                    ext = url.split(".")[-1].lower() if "." in url.split("/")[-1] else "unknown"
-                    if ext in ["jpg", "jpeg", "png", "gif", "webp", "svg", "ico"]:
-                        resource_type = "image"
-                    elif ext in ["js"]:
-                        resource_type = "javascript"
-                    elif ext in ["css"]:
-                        resource_type = "css"
-                    elif ext in ["woff", "woff2", "ttf", "otf", "eot"]:
-                        resource_type = "font"
-                    else:
-                        resource_type = "other"
-                    
-                    if resource_type not in resource_timings:
-                        resource_timings[resource_type] = []
-                    
-                    # Calculate request duration if timing information is available
-                    if isinstance(timing, dict) and "requestTime" in timing and "receiveHeadersEnd" in timing:
-                        # Convert to milliseconds
-                        duration = (timing["receiveHeadersEnd"] - timing["requestTime"]) * 1000
-                        resource_timings[resource_type].append({
-                            "url": url,
-                            "duration_ms": duration
-                        })
-                    if isinstance(timing, dict) and "requestStart" in timing and "responseStart" in timing and "startTime" in timing:
-                        # Convert to milliseconds
-                        duration = (timing["responseStart"] - timing["requestStart"]) * 1000
-                        resource_timings[resource_type].append({
-                            "url": url,
-                            "duration_ms": duration
-                        })
-                
-                # Calculate statistics for each resource type
-                print("\nPerformance by resource type:")
-                for resource_type, timings in resource_timings.items():
-                    if timings:
-                        durations = [t["duration_ms"] for t in timings]
-                        avg_duration = sum(durations) / len(durations)
-                        max_duration = max(durations)
-                        slowest_resource = next(t["url"] for t in timings if t["duration_ms"] == max_duration)
-                        
-                        print(f"  {resource_type.upper()}:")
-                        print(f"    - Count: {len(timings)}")
-                        print(f"    - Avg time: {avg_duration:.2f} ms")
-                        print(f"    - Max time: {max_duration:.2f} ms")
-                        print(f"    - Slowest: {slowest_resource}")
-                
-                # Identify the slowest resources overall
-                all_timings = []
-                for resource_type, timings in resource_timings.items():
-                    for timing in timings:
-                        timing["type"] = resource_type
-                        all_timings.append(timing)
-                
-                all_timings.sort(key=lambda x: x["duration_ms"], reverse=True)
-                
-                print("\nTop 5 slowest resources:")
-                for i, timing in enumerate(all_timings[:5], 1):
-                    print(f"  {i}. [{timing['type']}] {timing['url']} - {timing['duration_ms']:.2f} ms")
-                
-                # Save performance analysis to file
-                output_file = os.path.join(__cur_dir__, "tmp", "performance_analysis.json")
-                with open(output_file, "w") as f:
-                    json.dump({
-                        "url": result.url,
-                        "timestamp": datetime.now().isoformat(),
-                        "resource_timings": resource_timings,
-                        "slowest_resources": all_timings[:10]  # Save top 10
-                    }, f, indent=2)
-                
-                print(f"\nFull performance analysis saved to {output_file}")
-
-async def main():
-    """Run all demo functions sequentially"""
-    print("=== Network and Console Capture Examples ===")
-    
-    # Make sure tmp directory exists
-    os.makedirs(os.path.join(__cur_dir__, "tmp"), exist_ok=True)
-    
-    # Run basic examples
-    # await demo_basic_network_capture()
-    await demo_basic_console_capture()
-    # await demo_combined_capture()
-    
-    # Run advanced examples
-    # await analyze_spa_network_traffic()
-    # await demo_security_analysis()
-    # await demo_performance_analysis()
-    
-    print("\n=== Examples Complete ===")
-    print(f"Check the tmp directory for output files: {os.path.join(__cur_dir__, 'tmp')}")
-
-if __name__ == "__main__":
-    asyncio.run(main())

docs/examples/quickstart_async.py

@@ -0,0 +1,675 @@
+import os, sys
+
+from crawl4ai import LLMConfig
+
+# append parent directory to system path
+sys.path.append(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+)
+os.environ["FIRECRAWL_API_KEY"] = "fc-84b370ccfad44beabc686b38f1769692"
+
+import asyncio
+# import nest_asyncio
+# nest_asyncio.apply()
+
+import time
+import json
+import os
+import re
+from typing import Dict, List
+from bs4 import BeautifulSoup
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.extraction_strategy import (
+    JsonCssExtractionStrategy,
+    LLMExtractionStrategy,
+)
+
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+print("Crawl4AI: Advanced Web Crawling and Data Extraction")
+print("GitHub Repository: https://github.com/unclecode/crawl4ai")
+print("Twitter: @unclecode")
+print("Website: https://crawl4ai.com")
+
+
+async def simple_crawl():
+    print("\n--- Basic Usage ---")
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", cache_mode=CacheMode.BYPASS
+        )
+        print(result.markdown[:500])  # Print first 500 characters
+
+
+async def simple_example_with_running_js_code():
+    print("\n--- Executing JavaScript and Using CSS Selectors ---")
+    # New code to handle the wait_for parameter
+    wait_for = """() => {
+        return Array.from(document.querySelectorAll('article.tease-card')).length > 10;
+    }"""
+
+    # wait_for can be also just a css selector
+    # wait_for = "article.tease-card:nth-child(10)"
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        js_code = [
+            "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+        ]
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            js_code=js_code,
+            # wait_for=wait_for,
+            cache_mode=CacheMode.BYPASS,
+        )
+        print(result.markdown[:500])  # Print first 500 characters
+
+
+async def simple_example_with_css_selector():
+    print("\n--- Using CSS Selectors ---")
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            css_selector=".wide-tease-item__description",
+            cache_mode=CacheMode.BYPASS,
+        )
+        print(result.markdown[:500])  # Print first 500 characters
+
+
+async def use_proxy():
+    print("\n--- Using a Proxy ---")
+    print(
+        "Note: Replace 'http://your-proxy-url:port' with a working proxy to run this example."
+    )
+    # Uncomment and modify the following lines to use a proxy
+    async with AsyncWebCrawler(
+        verbose=True, proxy="http://your-proxy-url:port"
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business", cache_mode=CacheMode.BYPASS
+        )
+        if result.success:
+            print(result.markdown[:500])  # Print first 500 characters
+
+
+async def capture_and_save_screenshot(url: str, output_path: str):
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url=url, screenshot=True, cache_mode=CacheMode.BYPASS
+        )
+
+        if result.success and result.screenshot:
+            import base64
+
+            # Decode the base64 screenshot data
+            screenshot_data = base64.b64decode(result.screenshot)
+
+            # Save the screenshot as a JPEG file
+            with open(output_path, "wb") as f:
+                f.write(screenshot_data)
+
+            print(f"Screenshot saved successfully to {output_path}")
+        else:
+            print("Failed to capture screenshot")
+
+
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(
+        ..., description="Fee for output token for the OpenAI model."
+    )
+
+
+async def extract_structured_data_using_llm(
+    provider: str, api_token: str = None, extra_headers: Dict[str, str] = None
+):
+    print(f"\n--- Extracting Structured Data with {provider} ---")
+
+    if api_token is None and provider != "ollama":
+        print(f"API token is required for {provider}. Skipping this example.")
+        return
+
+    # extra_args = {}
+    extra_args = {
+        "temperature": 0,
+        "top_p": 0.9,
+        "max_tokens": 2000,
+        # any other supported parameters for litellm
+    }
+    if extra_headers:
+        extra_args["extra_headers"] = extra_headers
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        result = await crawler.arun(
+            url="https://openai.com/api/pricing/",
+            word_count_threshold=1,
+            extraction_strategy=LLMExtractionStrategy(
+                llm_config=LLMConfig(provider=provider,api_token=api_token),
+                schema=OpenAIModelFee.model_json_schema(),
+                extraction_type="schema",
+                instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+                Do not miss any models in the entire content. One extracted model JSON format should look like this: 
+                {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}.""",
+                extra_args=extra_args,
+            ),
+            cache_mode=CacheMode.BYPASS,
+        )
+        print(result.extracted_content)
+
+
+async def extract_structured_data_using_css_extractor():
+    print("\n--- Using JsonCssExtractionStrategy for Fast Structured Output ---")
+    schema = {
+        "name": "KidoCode Courses",
+        "baseSelector": "section.charge-methodology .w-tab-content > div",
+        "fields": [
+            {
+                "name": "section_title",
+                "selector": "h3.heading-50",
+                "type": "text",
+            },
+            {
+                "name": "section_description",
+                "selector": ".charge-content",
+                "type": "text",
+            },
+            {
+                "name": "course_name",
+                "selector": ".text-block-93",
+                "type": "text",
+            },
+            {
+                "name": "course_description",
+                "selector": ".course-content-text",
+                "type": "text",
+            },
+            {
+                "name": "course_icon",
+                "selector": ".image-92",
+                "type": "attribute",
+                "attribute": "src",
+            },
+        ],
+    }
+
+    async with AsyncWebCrawler(headless=True, verbose=True) as crawler:
+        # Create the JavaScript that handles clicking multiple times
+        js_click_tabs = """
+        (async () => {
+            const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");
+            
+            for(let tab of tabs) {
+                // scroll to the tab
+                tab.scrollIntoView();
+                tab.click();
+                // Wait for content to load and animations to complete
+                await new Promise(r => setTimeout(r, 500));
+            }
+        })();
+        """
+
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            extraction_strategy=JsonCssExtractionStrategy(schema, verbose=True),
+            js_code=[js_click_tabs],
+            cache_mode=CacheMode.BYPASS,
+        )
+
+        companies = json.loads(result.extracted_content)
+        print(f"Successfully extracted {len(companies)} companies")
+        print(json.dumps(companies[0], indent=2))
+
+
+# Advanced Session-Based Crawling with Dynamic Content 🔄
+async def crawl_dynamic_content_pages_method_1():
+    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
+    first_commit = ""
+
+    async def on_execution_started(page):
+        nonlocal first_commit
+        try:
+            while True:
+                await page.wait_for_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await page.query_selector("li.Box-sc-g0xbh4-0 h4")
+                commit = await commit.evaluate("(element) => element.textContent")
+                commit = re.sub(r"\s+", "", commit)
+                if commit and commit != first_commit:
+                    first_commit = commit
+                    break
+                await asyncio.sleep(0.5)
+        except Exception as e:
+            print(f"Warning: New content didn't appear after JavaScript execution: {e}")
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        js_next_page = """
+        (() => {
+            const button = document.querySelector('a[data-testid="pagination-next-button"]');
+            if (button) button.click();
+        })();
+        """
+
+        for page in range(3):  # Crawl 3 pages
+            result = await crawler.arun(
+                url=url,
+                session_id=session_id,
+                css_selector="li.Box-sc-g0xbh4-0",
+                js=js_next_page if page > 0 else None,
+                cache_mode=CacheMode.BYPASS,
+                js_only=page > 0,
+                headless=False,
+            )
+
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            soup = BeautifulSoup(result.cleaned_html, "html.parser")
+            commits = soup.select("li")
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+async def crawl_dynamic_content_pages_method_2():
+    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+        last_commit = ""
+
+        js_next_page_and_wait = """
+        (async () => {
+            const getCurrentCommit = () => {
+                const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+                return commits.length > 0 ? commits[0].textContent.trim() : null;
+            };
+
+            const initialCommit = getCurrentCommit();
+            const button = document.querySelector('a[data-testid="pagination-next-button"]');
+            if (button) button.click();
+
+            // Poll for changes
+            while (true) {
+                await new Promise(resolve => setTimeout(resolve, 100)); // Wait 100ms
+                const newCommit = getCurrentCommit();
+                if (newCommit && newCommit !== initialCommit) {
+                    break;
+                }
+            }
+        })();
+        """
+
+        schema = {
+            "name": "Commit Extractor",
+            "baseSelector": "li.Box-sc-g0xbh4-0",
+            "fields": [
+                {
+                    "name": "title",
+                    "selector": "h4.markdown-title",
+                    "type": "text",
+                    "transform": "strip",
+                },
+            ],
+        }
+        extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+        for page in range(3):  # Crawl 3 pages
+            result = await crawler.arun(
+                url=url,
+                session_id=session_id,
+                css_selector="li.Box-sc-g0xbh4-0",
+                extraction_strategy=extraction_strategy,
+                js_code=js_next_page_and_wait if page > 0 else None,
+                js_only=page > 0,
+                cache_mode=CacheMode.BYPASS,
+                headless=False,
+            )
+
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            commits = json.loads(result.extracted_content)
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+async def crawl_dynamic_content_pages_method_3():
+    print(
+        "\n--- Advanced Multi-Page Crawling with JavaScript Execution using `wait_for` ---"
+    )
+
+    async with AsyncWebCrawler(verbose=True) as crawler:
+        url = "https://github.com/microsoft/TypeScript/commits/main"
+        session_id = "typescript_commits_session"
+        all_commits = []
+
+        js_next_page = """
+        const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+        if (commits.length > 0) {
+            window.firstCommit = commits[0].textContent.trim();
+        }
+        const button = document.querySelector('a[data-testid="pagination-next-button"]');
+        if (button) button.click();
+        """
+
+        wait_for = """() => {
+            const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
+            if (commits.length === 0) return false;
+            const firstCommit = commits[0].textContent.trim();
+            return firstCommit !== window.firstCommit;
+        }"""
+
+        schema = {
+            "name": "Commit Extractor",
+            "baseSelector": "li.Box-sc-g0xbh4-0",
+            "fields": [
+                {
+                    "name": "title",
+                    "selector": "h4.markdown-title",
+                    "type": "text",
+                    "transform": "strip",
+                },
+            ],
+        }
+        extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+        for page in range(3):  # Crawl 3 pages
+            result = await crawler.arun(
+                url=url,
+                session_id=session_id,
+                css_selector="li.Box-sc-g0xbh4-0",
+                extraction_strategy=extraction_strategy,
+                js_code=js_next_page if page > 0 else None,
+                wait_for=wait_for if page > 0 else None,
+                js_only=page > 0,
+                cache_mode=CacheMode.BYPASS,
+                headless=False,
+            )
+
+            assert result.success, f"Failed to crawl page {page + 1}"
+
+            commits = json.loads(result.extracted_content)
+            all_commits.extend(commits)
+
+            print(f"Page {page + 1}: Found {len(commits)} commits")
+
+        await crawler.crawler_strategy.kill_session(session_id)
+        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
+
+
+async def crawl_custom_browser_type():
+    # Use Firefox
+    start = time.time()
+    async with AsyncWebCrawler(
+        browser_type="firefox", verbose=True, headless=True
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com", cache_mode=CacheMode.BYPASS
+        )
+        print(result.markdown[:500])
+        print("Time taken: ", time.time() - start)
+
+    # Use WebKit
+    start = time.time()
+    async with AsyncWebCrawler(
+        browser_type="webkit", verbose=True, headless=True
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com", cache_mode=CacheMode.BYPASS
+        )
+        print(result.markdown[:500])
+        print("Time taken: ", time.time() - start)
+
+    # Use Chromium (default)
+    start = time.time()
+    async with AsyncWebCrawler(verbose=True, headless=True) as crawler:
+        result = await crawler.arun(
+            url="https://www.example.com", cache_mode=CacheMode.BYPASS
+        )
+        print(result.markdown[:500])
+        print("Time taken: ", time.time() - start)
+
+
+async def crawl_with_user_simultion():
+    async with AsyncWebCrawler(verbose=True, headless=True) as crawler:
+        url = "YOUR-URL-HERE"
+        result = await crawler.arun(
+            url=url,
+            cache_mode=CacheMode.BYPASS,
+            magic=True,  # Automatically detects and removes overlays, popups, and other elements that block content
+            # simulate_user = True,# Causes a series of random mouse movements and clicks to simulate user interaction
+            # override_navigator = True # Overrides the navigator object to make it look like a real user
+        )
+
+        print(result.markdown)
+
+
+async def speed_comparison():
+    # print("\n--- Speed Comparison ---")
+    # print("Firecrawl (simulated):")
+    # print("Time taken: 7.02 seconds")
+    # print("Content length: 42074 characters")
+    # print("Images found: 49")
+    # print()
+    # Simulated Firecrawl performance
+    from firecrawl import FirecrawlApp
+
+    app = FirecrawlApp(api_key=os.environ["FIRECRAWL_API_KEY"])
+    start = time.time()
+    scrape_status = app.scrape_url(
+        "https://www.nbcnews.com/business", params={"formats": ["markdown", "html"]}
+    )
+    end = time.time()
+    print("Firecrawl:")
+    print(f"Time taken: {end - start:.2f} seconds")
+    print(f"Content length: {len(scrape_status['markdown'])} characters")
+    print(f"Images found: {scrape_status['markdown'].count('cldnry.s-nbcnews.com')}")
+    print()
+
+    async with AsyncWebCrawler() as crawler:
+        # Crawl4AI simple crawl
+        start = time.time()
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            word_count_threshold=0,
+            cache_mode=CacheMode.BYPASS,
+            verbose=False,
+        )
+        end = time.time()
+        print("Crawl4AI (simple crawl):")
+        print(f"Time taken: {end - start:.2f} seconds")
+        print(f"Content length: {len(result.markdown)} characters")
+        print(f"Images found: {result.markdown.count('cldnry.s-nbcnews.com')}")
+        print()
+
+        # Crawl4AI with advanced content filtering
+        start = time.time()
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            word_count_threshold=0,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48, threshold_type="fixed", min_word_threshold=0
+                )
+                # content_filter=BM25ContentFilter(user_query=None, bm25_threshold=1.0)
+            ),
+            cache_mode=CacheMode.BYPASS,
+            verbose=False,
+        )
+        end = time.time()
+        print("Crawl4AI (Markdown Plus):")
+        print(f"Time taken: {end - start:.2f} seconds")
+        print(f"Content length: {len(result.markdown.raw_markdown)} characters")
+        print(f"Fit Markdown: {len(result.markdown.fit_markdown)} characters")
+        print(f"Images found: {result.markdown.raw_markdown.count('cldnry.s-nbcnews.com')}")
+        print()
+
+        # Crawl4AI with JavaScript execution
+        start = time.time()
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+            js_code=[
+                "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+            ],
+            word_count_threshold=0,
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48, threshold_type="fixed", min_word_threshold=0
+                )
+                # content_filter=BM25ContentFilter(user_query=None, bm25_threshold=1.0)
+            ),
+            verbose=False,
+        )
+        end = time.time()
+        print("Crawl4AI (with JavaScript execution):")
+        print(f"Time taken: {end - start:.2f} seconds")
+        print(f"Content length: {len(result.markdown.raw_markdown)} characters")
+        print(f"Fit Markdown: {len(result.markdown.fit_markdown)} characters")
+        print(f"Images found: {result.markdown.raw_markdown.count('cldnry.s-nbcnews.com')}")
+
+    print("\nNote on Speed Comparison:")
+    print("The speed test conducted here may not reflect optimal conditions.")
+    print("When we call Firecrawl's API, we're seeing its best performance,")
+    print("while Crawl4AI's performance is limited by the local network speed.")
+    print("For a more accurate comparison, it's recommended to run these tests")
+    print("on servers with a stable and fast internet connection.")
+    print("Despite these limitations, Crawl4AI still demonstrates faster performance.")
+    print("If you run these tests in an environment with better network conditions,")
+    print("you may observe an even more significant speed advantage for Crawl4AI.")
+
+
+async def generate_knowledge_graph():
+    class Entity(BaseModel):
+        name: str
+        description: str
+
+    class Relationship(BaseModel):
+        entity1: Entity
+        entity2: Entity
+        description: str
+        relation_type: str
+
+    class KnowledgeGraph(BaseModel):
+        entities: List[Entity]
+        relationships: List[Relationship]
+
+    extraction_strategy = LLMExtractionStrategy(
+        llm_config=LLMConfig(provider="openai/gpt-4o-mini",  api_token=os.getenv("OPENAI_API_KEY")),  # In case of Ollama just pass "no-token"
+        schema=KnowledgeGraph.model_json_schema(),
+        extraction_type="schema",
+        instruction="""Extract entities and relationships from the given text.""",
+    )
+    async with AsyncWebCrawler() as crawler:
+        url = "https://paulgraham.com/love.html"
+        result = await crawler.arun(
+            url=url,
+            cache_mode=CacheMode.BYPASS,
+            extraction_strategy=extraction_strategy,
+            # magic=True
+        )
+        # print(result.extracted_content)
+        with open(os.path.join(__location__, "kb.json"), "w") as f:
+            f.write(result.extracted_content)
+
+
+async def fit_markdown_remove_overlay():
+    async with AsyncWebCrawler(
+        headless=True,  # Set to False to see what is happening
+        verbose=True,
+        user_agent_mode="random",
+        user_agent_generator_config={"device_type": "mobile", "os_type": "android"},
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            cache_mode=CacheMode.BYPASS,
+            markdown_generator=DefaultMarkdownGenerator(
+                content_filter=PruningContentFilter(
+                    threshold=0.48, threshold_type="fixed", min_word_threshold=0
+                ),
+                options={"ignore_links": True},
+            ),
+            # markdown_generator=DefaultMarkdownGenerator(
+            #     content_filter=BM25ContentFilter(user_query="", bm25_threshold=1.0),
+            #     options={
+            #         "ignore_links": True
+            #     }
+            # ),
+        )
+
+        if result.success:
+            print(len(result.markdown.raw_markdown))
+            print(len(result.markdown.markdown_with_citations))
+            print(len(result.markdown.fit_markdown))
+
+            # Save clean html
+            with open(os.path.join(__location__, "output/cleaned_html.html"), "w") as f:
+                f.write(result.cleaned_html)
+
+            with open(
+                os.path.join(__location__, "output/output_raw_markdown.md"), "w"
+            ) as f:
+                f.write(result.markdown.raw_markdown)
+
+            with open(
+                os.path.join(__location__, "output/output_markdown_with_citations.md"),
+                "w",
+            ) as f:
+                f.write(result.markdown.markdown_with_citations)
+
+            with open(
+                os.path.join(__location__, "output/output_fit_markdown.md"), "w"
+            ) as f:
+                f.write(result.markdown.fit_markdown)
+
+    print("Done")
+
+
+async def main():
+    # await extract_structured_data_using_llm("openai/gpt-4o", os.getenv("OPENAI_API_KEY"))
+
+    # await simple_crawl()
+    # await simple_example_with_running_js_code()
+    # await simple_example_with_css_selector()
+    # # await use_proxy()
+    # await capture_and_save_screenshot("https://www.example.com", os.path.join(__location__, "tmp/example_screenshot.jpg"))
+    # await extract_structured_data_using_css_extractor()
+
+    # LLM extraction examples
+    # await extract_structured_data_using_llm()
+    # await extract_structured_data_using_llm("huggingface/meta-llama/Meta-Llama-3.1-8B-Instruct", os.getenv("HUGGINGFACE_API_KEY"))
+    # await extract_structured_data_using_llm("ollama/llama3.2")
+
+    # You always can pass custom headers to the extraction strategy
+    # custom_headers = {
+    #     "Authorization": "Bearer your-custom-token",
+    #     "X-Custom-Header": "Some-Value"
+    # }
+    # await extract_structured_data_using_llm(extra_headers=custom_headers)
+
+    # await crawl_dynamic_content_pages_method_1()
+    # await crawl_dynamic_content_pages_method_2()
+    await crawl_dynamic_content_pages_method_3()
+
+    # await crawl_custom_browser_type()
+
+    # await speed_comparison()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/quickstart_examples_set_1.py

@@ -1,412 +0,0 @@
-import asyncio
-import os
-import json
-import base64
-from pathlib import Path
-from typing import List
-from crawl4ai.proxy_strategy import ProxyConfig
-
-from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode, CrawlResult
-from crawl4ai import RoundRobinProxyStrategy
-from crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategy
-from crawl4ai import LLMConfig
-from crawl4ai import PruningContentFilter, BM25ContentFilter
-from crawl4ai import DefaultMarkdownGenerator
-from crawl4ai import BFSDeepCrawlStrategy, DomainFilter, FilterChain
-from crawl4ai import BrowserConfig
-
-__cur_dir__ = Path(__file__).parent
-
-async def demo_basic_crawl():
-    """Basic web crawling with markdown generation"""
-    print("\n=== 1. Basic Web Crawling ===")
-    async with AsyncWebCrawler(config = BrowserConfig(
-        viewport_height=800,
-        viewport_width=1200,
-        headless=True,
-        verbose=True,
-    )) as crawler:
-        results: List[CrawlResult] = await crawler.arun(
-            url="https://news.ycombinator.com/"
-        )
-
-        for i, result in enumerate(results):
-            print(f"Result {i + 1}:")
-            print(f"Success: {result.success}")
-            if result.success:
-                print(f"Markdown length: {len(result.markdown.raw_markdown)} chars")
-                print(f"First 100 chars: {result.markdown.raw_markdown[:100]}...")
-            else:
-                print("Failed to crawl the URL")
-
-async def demo_parallel_crawl():
-    """Crawl multiple URLs in parallel"""
-    print("\n=== 2. Parallel Crawling ===")
-
-    urls = [
-        "https://news.ycombinator.com/",
-        "https://example.com/",
-        "https://httpbin.org/html",
-    ]
-
-    async with AsyncWebCrawler() as crawler:
-        results: List[CrawlResult] = await crawler.arun_many(
-            urls=urls,
-        )
-
-        print(f"Crawled {len(results)} URLs in parallel:")
-        for i, result in enumerate(results):
-            print(
-                f"  {i + 1}. {result.url} - {'Success' if result.success else 'Failed'}"
-            )
-
-async def demo_fit_markdown():
-    """Generate focused markdown with LLM content filter"""
-    print("\n=== 3. Fit Markdown with LLM Content Filter ===")
-
-    async with AsyncWebCrawler() as crawler:
-        result: CrawlResult = await crawler.arun(
-            url = "https://en.wikipedia.org/wiki/Python_(programming_language)",
-            config=CrawlerRunConfig(
-                markdown_generator=DefaultMarkdownGenerator(
-                    content_filter=PruningContentFilter()
-                )
-            ),
-        )
-
-        # Print stats and save the fit markdown
-        print(f"Raw: {len(result.markdown.raw_markdown)} chars")
-        print(f"Fit: {len(result.markdown.fit_markdown)} chars")
-
-async def demo_llm_structured_extraction_no_schema():
-    # Create a simple LLM extraction strategy (no schema required)
-    extraction_strategy = LLMExtractionStrategy(
-        llm_config=LLMConfig(
-            provider="groq/qwen-2.5-32b",
-            api_token="env:GROQ_API_KEY",
-        ),
-        instruction="This is news.ycombinator.com, extract all news, and for each, I want title, source url, number of comments.",
-        extract_type="schema",
-        schema="{title: string, url: string, comments: int}",
-        extra_args={
-            "temperature": 0.0,
-            "max_tokens": 4096,
-        },
-        verbose=True,
-    )
-
-    config = CrawlerRunConfig(extraction_strategy=extraction_strategy)
-
-    async with AsyncWebCrawler() as crawler:
-        results: List[CrawlResult] = await crawler.arun(
-            "https://news.ycombinator.com/", config=config
-        )
-
-        for result in results:
-            print(f"URL: {result.url}")
-            print(f"Success: {result.success}")
-            if result.success:
-                data = json.loads(result.extracted_content)
-                print(json.dumps(data, indent=2))
-            else:
-                print("Failed to extract structured data")
-
-async def demo_css_structured_extraction_no_schema():
-    """Extract structured data using CSS selectors"""
-    print("\n=== 5. CSS-Based Structured Extraction ===")
-    # Sample HTML for schema generation (one-time cost)
-    sample_html = """
-<div class="body-post clear">
-    <a class="story-link" href="https://thehackernews.com/2025/04/malicious-python-packages-on-pypi.html">
-        <div class="clear home-post-box cf">
-            <div class="home-img clear">
-                <div class="img-ratio">
-                    <img alt="..." src="...">
-                </div>
-            </div>
-            <div class="clear home-right">
-                <h2 class="home-title">Malicious Python Packages on PyPI Downloaded 39,000+ Times, Steal Sensitive Data</h2>
-                <div class="item-label">
-                    <span class="h-datetime"><i class="icon-font icon-calendar"></i>Apr 05, 2025</span>
-                    <span class="h-tags">Malware / Supply Chain Attack</span>
-                </div>
-                <div class="home-desc"> Cybersecurity researchers have...</div>
-            </div>
-        </div>
-    </a>
-</div>
-    """
-
-    # Check if schema file exists
-    schema_file_path = f"{__cur_dir__}/tmp/schema.json"
-    if os.path.exists(schema_file_path):
-        with open(schema_file_path, "r") as f:
-            schema = json.load(f)
-    else:
-        # Generate schema using LLM (one-time setup)
-        schema = JsonCssExtractionStrategy.generate_schema(
-            html=sample_html,
-            llm_config=LLMConfig(
-                provider="groq/qwen-2.5-32b",
-                api_token="env:GROQ_API_KEY",
-            ),
-            query="From https://thehackernews.com/, I have shared a sample of one news div with a title, date, and description. Please generate a schema for this news div.",
-        )
-
-    print(f"Generated schema: {json.dumps(schema, indent=2)}")
-    # Save the schema to a file , and use it for future extractions, in result for such extraction you will call LLM once
-    with open(f"{__cur_dir__}/tmp/schema.json", "w") as f:
-        json.dump(schema, f, indent=2)
-
-    # Create no-LLM extraction strategy with the generated schema
-    extraction_strategy = JsonCssExtractionStrategy(schema)
-    config = CrawlerRunConfig(extraction_strategy=extraction_strategy)
-
-    # Use the fast CSS extraction (no LLM calls during extraction)
-    async with AsyncWebCrawler() as crawler:
-        results: List[CrawlResult] = await crawler.arun(
-            "https://thehackernews.com", config=config
-        )
-
-        for result in results:
-            print(f"URL: {result.url}")
-            print(f"Success: {result.success}")
-            if result.success:
-                data = json.loads(result.extracted_content)
-                print(json.dumps(data, indent=2))
-            else:
-                print("Failed to extract structured data")
-
-async def demo_deep_crawl():
-    """Deep crawling with BFS strategy"""
-    print("\n=== 6. Deep Crawling ===")
-
-    filter_chain = FilterChain([DomainFilter(allowed_domains=["crawl4ai.com"])])
-
-    deep_crawl_strategy = BFSDeepCrawlStrategy(
-        max_depth=1, max_pages=5, filter_chain=filter_chain
-    )
-
-    async with AsyncWebCrawler() as crawler:
-        results: List[CrawlResult] = await crawler.arun(
-            url="https://docs.crawl4ai.com",
-            config=CrawlerRunConfig(deep_crawl_strategy=deep_crawl_strategy),
-        )
-
-        print(f"Deep crawl returned {len(results)} pages:")
-        for i, result in enumerate(results):
-            depth = result.metadata.get("depth", "unknown")
-            print(f"  {i + 1}. {result.url} (Depth: {depth})")
-
-async def demo_js_interaction():
-    """Execute JavaScript to load more content"""
-    print("\n=== 7. JavaScript Interaction ===")
-
-    # A simple page that needs JS to reveal content
-    async with AsyncWebCrawler(config=BrowserConfig(headless=False)) as crawler:
-        # Initial load
-
-        news_schema = {
-            "name": "news",
-            "baseSelector": "tr.athing",
-            "fields": [
-                {
-                    "name": "title",
-                    "selector": "span.titleline",
-                    "type": "text",
-                }
-            ],
-        }
-        results: List[CrawlResult] = await crawler.arun(
-            url="https://news.ycombinator.com",
-            config=CrawlerRunConfig(
-                session_id="hn_session",  # Keep session
-                extraction_strategy=JsonCssExtractionStrategy(schema=news_schema),
-            ),
-        )
-
-        news = []
-        for result in results:
-            if result.success:
-                data = json.loads(result.extracted_content)
-                news.extend(data)
-                print(json.dumps(data, indent=2))
-            else:
-                print("Failed to extract structured data")
-
-        print(f"Initial items: {len(news)}")
-
-        # Click "More" link
-        more_config = CrawlerRunConfig(
-            js_code="document.querySelector('a.morelink').click();",
-            js_only=True,  # Continue in same page
-            session_id="hn_session",  # Keep session
-            extraction_strategy=JsonCssExtractionStrategy(
-                schema=news_schema,
-            ),
-        )
-
-        result: List[CrawlResult] = await crawler.arun(
-            url="https://news.ycombinator.com", config=more_config
-        )
-
-        # Extract new items
-        for result in results:
-            if result.success:
-                data = json.loads(result.extracted_content)
-                news.extend(data)
-                print(json.dumps(data, indent=2))
-            else:
-                print("Failed to extract structured data")
-        print(f"Total items: {len(news)}")
-
-async def demo_media_and_links():
-    """Extract media and links from a page"""
-    print("\n=== 8. Media and Links Extraction ===")
-
-    async with AsyncWebCrawler() as crawler:
-        result: List[CrawlResult] = await crawler.arun("https://en.wikipedia.org/wiki/Main_Page")
-
-        for i, result in enumerate(result):
-            # Extract and save all images
-            images = result.media.get("images", [])
-            print(f"Found {len(images)} images")
-
-            # Extract and save all links (internal and external)
-            internal_links = result.links.get("internal", [])
-            external_links = result.links.get("external", [])
-            print(f"Found {len(internal_links)} internal links")
-            print(f"Found {len(external_links)} external links")
-
-            # Print some of the images and links
-            for image in images[:3]:
-                print(f"Image: {image['src']}")
-            for link in internal_links[:3]:
-                print(f"Internal link: {link['href']}")
-            for link in external_links[:3]:
-                print(f"External link: {link['href']}")
-
-            # # Save everything to files
-            with open(f"{__cur_dir__}/tmp/images.json", "w") as f:
-                json.dump(images, f, indent=2)
-
-            with open(f"{__cur_dir__}/tmp/links.json", "w") as f:
-                json.dump(
-                    {"internal": internal_links, "external": external_links},
-                    f,
-                    indent=2,
-                )
-
-async def demo_screenshot_and_pdf():
-    """Capture screenshot and PDF of a page"""
-    print("\n=== 9. Screenshot and PDF Capture ===")
-
-    async with AsyncWebCrawler() as crawler:
-        result: List[CrawlResult] = await crawler.arun(
-            # url="https://example.com",
-            url="https://en.wikipedia.org/wiki/Giant_anteater",
-            config=CrawlerRunConfig(screenshot=True, pdf=True),
-        )
-
-        for i, result in enumerate(result):
-            # if result.screenshot_data:
-            if result.screenshot:
-                # Save screenshot
-                screenshot_path = f"{__cur_dir__}/tmp/example_screenshot.png"
-                with open(screenshot_path, "wb") as f:
-                    f.write(base64.b64decode(result.screenshot))
-                print(f"Screenshot saved to {screenshot_path}")
-
-            # if result.pdf_data:
-            if result.pdf:
-                # Save PDF
-                pdf_path = f"{__cur_dir__}/tmp/example.pdf"
-                with open(pdf_path, "wb") as f:
-                    f.write(result.pdf)
-                print(f"PDF saved to {pdf_path}")
-
-async def demo_proxy_rotation():
-    """Proxy rotation for multiple requests"""
-    print("\n=== 10. Proxy Rotation ===")
-
-    # Example proxies (replace with real ones)
-    proxies = [
-        ProxyConfig(server="http://proxy1.example.com:8080"),
-        ProxyConfig(server="http://proxy2.example.com:8080"),
-    ]
-
-    proxy_strategy = RoundRobinProxyStrategy(proxies)
-
-    print(f"Using {len(proxies)} proxies in rotation")
-    print(
-        "Note: This example uses placeholder proxies - replace with real ones to test"
-    )
-
-    async with AsyncWebCrawler() as crawler:
-        config = CrawlerRunConfig(
-            proxy_rotation_strategy=proxy_strategy
-        )
-
-        # In a real scenario, these would be run and the proxies would rotate
-        print("In a real scenario, requests would rotate through the available proxies")
-
-async def demo_raw_html_and_file():
-    """Process raw HTML and local files"""
-    print("\n=== 11. Raw HTML and Local Files ===")
-
-    raw_html = """
-    <html><body>
-        <h1>Sample Article</h1>
-        <p>This is sample content for testing Crawl4AI's raw HTML processing.</p>
-    </body></html>
-    """
-
-    # Save to file
-    file_path = Path("docs/examples/tmp/sample.html").absolute()
-    with open(file_path, "w") as f:
-        f.write(raw_html)
-
-    async with AsyncWebCrawler() as crawler:
-        # Crawl raw HTML
-        raw_result = await crawler.arun(
-            url="raw:" + raw_html, config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
-        )
-        print("Raw HTML processing:")
-        print(f"  Markdown: {raw_result.markdown.raw_markdown[:50]}...")
-
-        # Crawl local file
-        file_result = await crawler.arun(
-            url=f"file://{file_path}",
-            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
-        )
-        print("\nLocal file processing:")
-        print(f"  Markdown: {file_result.markdown.raw_markdown[:50]}...")
-
-    # Clean up
-    os.remove(file_path)
-    print(f"Processed both raw HTML and local file ({file_path})")
-
-async def main():
-    """Run all demo functions sequentially"""
-    print("=== Comprehensive Crawl4AI Demo ===")
-    print("Note: Some examples require API keys or other configurations")
-
-    # Run all demos
-    await demo_basic_crawl()
-    await demo_parallel_crawl()
-    await demo_fit_markdown()
-    await demo_llm_structured_extraction_no_schema()
-    await demo_css_structured_extraction_no_schema()
-    await demo_deep_crawl()
-    await demo_js_interaction()
-    await demo_media_and_links()
-    await demo_screenshot_and_pdf()
-    # # await demo_proxy_rotation()
-    await demo_raw_html_and_file()
-
-    # Clean up any temp files that may have been created
-    print("\n=== Demo Complete ===")
-    print("Check for any generated files (screenshots, PDFs) in the current directory")
-
-if __name__ == "__main__":
-    asyncio.run(main())

docs/examples/quickstart_examples_set_2.py

@@ -1,562 +0,0 @@
-import os, sys
-
-from crawl4ai.types import LLMConfig
-
-sys.path.append(
-    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-)
-
-import asyncio
-import time
-import json
-import re
-from typing import Dict
-from bs4 import BeautifulSoup
-from pydantic import BaseModel, Field
-from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
-from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-from crawl4ai.content_filter_strategy import PruningContentFilter
-from crawl4ai.extraction_strategy import (
-    JsonCssExtractionStrategy,
-    LLMExtractionStrategy,
-)
-
-__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
-
-print("Crawl4AI: Advanced Web Crawling and Data Extraction")
-print("GitHub Repository: https://github.com/unclecode/crawl4ai")
-print("Twitter: @unclecode")
-print("Website: https://crawl4ai.com")
-
-
-# Basic Example - Simple Crawl
-async def simple_crawl():
-    print("\n--- Basic Usage ---")
-    browser_config = BrowserConfig(headless=True)
-    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(
-            url="https://www.nbcnews.com/business", config=crawler_config
-        )
-        print(result.markdown[:500])
-
-
-async def clean_content():
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        excluded_tags=["nav", "footer", "aside"],
-        remove_overlay_elements=True,
-        markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(
-                threshold=0.48, threshold_type="fixed", min_word_threshold=0
-            ),
-            options={"ignore_links": True},
-        ),
-    )
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun(
-            url="https://en.wikipedia.org/wiki/Apple",
-            config=crawler_config,
-        )
-        full_markdown_length = len(result.markdown.raw_markdown)
-        fit_markdown_length = len(result.markdown.fit_markdown)
-        print(f"Full Markdown Length: {full_markdown_length}")
-        print(f"Fit Markdown Length: {fit_markdown_length}")
-
-
-async def link_analysis():
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.ENABLED,
-        exclude_external_links=True,
-        exclude_social_media_links=True,
-    )
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun(
-            url="https://www.nbcnews.com/business",
-            config=crawler_config,
-        )
-        print(f"Found {len(result.links['internal'])} internal links")
-        print(f"Found {len(result.links['external'])} external links")
-
-        for link in result.links["internal"][:5]:
-            print(f"Href: {link['href']}\nText: {link['text']}\n")
-
-
-# JavaScript Execution Example
-async def simple_example_with_running_js_code():
-    print("\n--- Executing JavaScript and Using CSS Selectors ---")
-
-    browser_config = BrowserConfig(headless=True, java_script_enabled=True)
-
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        js_code="const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();",
-        # wait_for="() => { return Array.from(document.querySelectorAll('article.tease-card')).length > 10; }"
-    )
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(
-            url="https://www.nbcnews.com/business", config=crawler_config
-        )
-        print(result.markdown[:500])
-
-
-# CSS Selector Example
-async def simple_example_with_css_selector():
-    print("\n--- Using CSS Selectors ---")
-    browser_config = BrowserConfig(headless=True)
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS, css_selector=".wide-tease-item__description"
-    )
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(
-            url="https://www.nbcnews.com/business", config=crawler_config
-        )
-        print(result.markdown[:500])
-
-
-async def media_handling():
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS, exclude_external_images=True, screenshot=True
-    )
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun(
-            url="https://www.nbcnews.com/business", config=crawler_config
-        )
-        for img in result.media["images"][:5]:
-            print(f"Image URL: {img['src']}, Alt: {img['alt']}, Score: {img['score']}")
-
-
-async def custom_hook_workflow(verbose=True):
-    async with AsyncWebCrawler() as crawler:
-        # Set a 'before_goto' hook to run custom code just before navigation
-        crawler.crawler_strategy.set_hook(
-            "before_goto",
-            lambda page, context: print("[Hook] Preparing to navigate..."),
-        )
-
-        # Perform the crawl operation
-        result = await crawler.arun(url="https://crawl4ai.com")
-        print(result.markdown.raw_markdown[:500].replace("\n", " -- "))
-
-
-# Proxy Example
-async def use_proxy():
-    print("\n--- Using a Proxy ---")
-    browser_config = BrowserConfig(
-        headless=True,
-        proxy_config={
-            "server": "http://proxy.example.com:8080",
-            "username": "username",
-            "password": "password",
-        },
-    )
-    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(
-            url="https://www.nbcnews.com/business", config=crawler_config
-        )
-        if result.success:
-            print(result.markdown[:500])
-
-
-# Screenshot Example
-async def capture_and_save_screenshot(url: str, output_path: str):
-    browser_config = BrowserConfig(headless=True)
-    crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, screenshot=True)
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(url=url, config=crawler_config)
-
-        if result.success and result.screenshot:
-            import base64
-
-            screenshot_data = base64.b64decode(result.screenshot)
-            with open(output_path, "wb") as f:
-                f.write(screenshot_data)
-            print(f"Screenshot saved successfully to {output_path}")
-        else:
-            print("Failed to capture screenshot")
-
-
-# LLM Extraction Example
-class OpenAIModelFee(BaseModel):
-    model_name: str = Field(..., description="Name of the OpenAI model.")
-    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
-    output_fee: str = Field(
-        ..., description="Fee for output token for the OpenAI model."
-    )
-
-
-async def extract_structured_data_using_llm(
-    provider: str, api_token: str = None, extra_headers: Dict[str, str] = None
-):
-    print(f"\n--- Extracting Structured Data with {provider} ---")
-
-    if api_token is None and provider != "ollama":
-        print(f"API token is required for {provider}. Skipping this example.")
-        return
-
-    browser_config = BrowserConfig(headless=True)
-
-    extra_args = {"temperature": 0, "top_p": 0.9, "max_tokens": 2000}
-    if extra_headers:
-        extra_args["extra_headers"] = extra_headers
-
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        word_count_threshold=1,
-        page_timeout=80000,
-        extraction_strategy=LLMExtractionStrategy(
-            llm_config=LLMConfig(provider=provider,api_token=api_token),
-            schema=OpenAIModelFee.model_json_schema(),
-            extraction_type="schema",
-            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
-            Do not miss any models in the entire content.""",
-            extra_args=extra_args,
-        ),
-    )
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(
-            url="https://openai.com/api/pricing/", config=crawler_config
-        )
-        print(result.extracted_content)
-
-
-# CSS Extraction Example
-async def extract_structured_data_using_css_extractor():
-    print("\n--- Using JsonCssExtractionStrategy for Fast Structured Output ---")
-    schema = {
-        "name": "KidoCode Courses",
-        "baseSelector": "section.charge-methodology .framework-collection-item.w-dyn-item",
-        "fields": [
-            {
-                "name": "section_title",
-                "selector": "h3.heading-50",
-                "type": "text",
-            },
-            {
-                "name": "section_description",
-                "selector": ".charge-content",
-                "type": "text",
-            },
-            {
-                "name": "course_name",
-                "selector": ".text-block-93",
-                "type": "text",
-            },
-            {
-                "name": "course_description",
-                "selector": ".course-content-text",
-                "type": "text",
-            },
-            {
-                "name": "course_icon",
-                "selector": ".image-92",
-                "type": "attribute",
-                "attribute": "src",
-            },
-        ],
-    }
-
-    browser_config = BrowserConfig(headless=True, java_script_enabled=True)
-
-    js_click_tabs = """
-    (async () => {
-        const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");
-        for(let tab of tabs) {
-            tab.scrollIntoView();
-            tab.click();
-            await new Promise(r => setTimeout(r, 500));
-        }
-    })();
-    """
-
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        extraction_strategy=JsonCssExtractionStrategy(schema),
-        js_code=[js_click_tabs],
-        delay_before_return_html=1
-    )
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(
-            url="https://www.kidocode.com/degrees/technology", config=crawler_config
-        )
-
-        companies = json.loads(result.extracted_content)
-        print(f"Successfully extracted {len(companies)} companies")
-        print(json.dumps(companies[0], indent=2))
-
-
-# Dynamic Content Examples - Method 1
-async def crawl_dynamic_content_pages_method_1():
-    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
-    first_commit = ""
-
-    async def on_execution_started(page, **kwargs):
-        nonlocal first_commit
-        try:
-            while True:
-                await page.wait_for_selector("li.Box-sc-g0xbh4-0 h4")
-                commit = await page.query_selector("li.Box-sc-g0xbh4-0 h4")
-                commit = await commit.evaluate("(element) => element.textContent")
-                commit = re.sub(r"\s+", "", commit)
-                if commit and commit != first_commit:
-                    first_commit = commit
-                    break
-                await asyncio.sleep(0.5)
-        except Exception as e:
-            print(f"Warning: New content didn't appear after JavaScript execution: {e}")
-
-    browser_config = BrowserConfig(headless=False, java_script_enabled=True)
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
-
-        url = "https://github.com/microsoft/TypeScript/commits/main"
-        session_id = "typescript_commits_session"
-        all_commits = []
-
-        js_next_page = """
-        const button = document.querySelector('a[data-testid="pagination-next-button"]');
-        if (button) button.click();
-        """
-
-        for page in range(3):
-            crawler_config = CrawlerRunConfig(
-                cache_mode=CacheMode.BYPASS,
-                css_selector="li.Box-sc-g0xbh4-0",
-                js_code=js_next_page if page > 0 else None,
-                js_only=page > 0,
-                session_id=session_id,
-            )
-
-            result = await crawler.arun(url=url, config=crawler_config)
-            assert result.success, f"Failed to crawl page {page + 1}"
-
-            soup = BeautifulSoup(result.cleaned_html, "html.parser")
-            commits = soup.select("li")
-            all_commits.extend(commits)
-
-            print(f"Page {page + 1}: Found {len(commits)} commits")
-
-        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
-
-
-# Dynamic Content Examples - Method 2
-async def crawl_dynamic_content_pages_method_2():
-    print("\n--- Advanced Multi-Page Crawling with JavaScript Execution ---")
-
-    browser_config = BrowserConfig(headless=False, java_script_enabled=True)
-
-    js_next_page_and_wait = """
-    (async () => {
-        const getCurrentCommit = () => {
-            const commits = document.querySelectorAll('li.Box-sc-g0xbh4-0 h4');
-            return commits.length > 0 ? commits[0].textContent.trim() : null;
-        };
-
-        const initialCommit = getCurrentCommit();
-        const button = document.querySelector('a[data-testid="pagination-next-button"]');
-        if (button) button.click();
-
-        while (true) {
-            await new Promise(resolve => setTimeout(resolve, 100));
-            const newCommit = getCurrentCommit();
-            if (newCommit && newCommit !== initialCommit) {
-                break;
-            }
-        }
-    })();
-    """
-
-    schema = {
-        "name": "Commit Extractor",
-        "baseSelector": "li.Box-sc-g0xbh4-0",
-        "fields": [
-            {
-                "name": "title",
-                "selector": "h4.markdown-title",
-                "type": "text",
-                "transform": "strip",
-            },
-        ],
-    }
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        url = "https://github.com/microsoft/TypeScript/commits/main"
-        session_id = "typescript_commits_session"
-        all_commits = []
-
-        extraction_strategy = JsonCssExtractionStrategy(schema)
-
-        for page in range(3):
-            crawler_config = CrawlerRunConfig(
-                cache_mode=CacheMode.BYPASS,
-                css_selector="li.Box-sc-g0xbh4-0",
-                extraction_strategy=extraction_strategy,
-                js_code=js_next_page_and_wait if page > 0 else None,
-                js_only=page > 0,
-                session_id=session_id,
-            )
-
-            result = await crawler.arun(url=url, config=crawler_config)
-            assert result.success, f"Failed to crawl page {page + 1}"
-
-            commits = json.loads(result.extracted_content)
-            all_commits.extend(commits)
-            print(f"Page {page + 1}: Found {len(commits)} commits")
-
-        print(f"Successfully crawled {len(all_commits)} commits across 3 pages")
-
-
-async def cosine_similarity_extraction():
-    from crawl4ai.extraction_strategy import CosineStrategy
-    crawl_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        extraction_strategy=CosineStrategy(
-            word_count_threshold=10,
-            max_dist=0.2,  # Maximum distance between two words
-            linkage_method="ward",  # Linkage method for hierarchical clustering (ward, complete, average, single)
-            top_k=3,  # Number of top keywords to extract
-            sim_threshold=0.3,  # Similarity threshold for clustering
-            semantic_filter="McDonald's economic impact, American consumer trends",  # Keywords to filter the content semantically using embeddings
-            verbose=True,
-        ),
-    )
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun(
-            url="https://www.nbcnews.com/business/consumer/how-mcdonalds-e-coli-crisis-inflation-politics-reflect-american-story-rcna177156",
-            config=crawl_config,
-        )
-        print(json.loads(result.extracted_content)[:5])
-
-
-# Browser Comparison
-async def crawl_custom_browser_type():
-    print("\n--- Browser Comparison ---")
-
-    # Firefox
-    browser_config_firefox = BrowserConfig(browser_type="firefox", headless=True)
-    start = time.time()
-    async with AsyncWebCrawler(config=browser_config_firefox) as crawler:
-        result = await crawler.arun(
-            url="https://www.example.com",
-            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
-        )
-        print("Firefox:", time.time() - start)
-        print(result.markdown[:500])
-
-    # WebKit
-    browser_config_webkit = BrowserConfig(browser_type="webkit", headless=True)
-    start = time.time()
-    async with AsyncWebCrawler(config=browser_config_webkit) as crawler:
-        result = await crawler.arun(
-            url="https://www.example.com",
-            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
-        )
-        print("WebKit:", time.time() - start)
-        print(result.markdown[:500])
-
-    # Chromium (default)
-    browser_config_chromium = BrowserConfig(browser_type="chromium", headless=True)
-    start = time.time()
-    async with AsyncWebCrawler(config=browser_config_chromium) as crawler:
-        result = await crawler.arun(
-            url="https://www.example.com",
-            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
-        )
-        print("Chromium:", time.time() - start)
-        print(result.markdown[:500])
-
-
-# Anti-Bot and User Simulation
-async def crawl_with_user_simulation():
-    browser_config = BrowserConfig(
-        headless=True,
-        user_agent_mode="random",
-        user_agent_generator_config={"device_type": "mobile", "os_type": "android"},
-    )
-
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        magic=True,
-        simulate_user=True,
-        override_navigator=True,
-    )
-
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(url="YOUR-URL-HERE", config=crawler_config)
-        print(result.markdown)
-
-
-async def ssl_certification():
-    # Configure crawler to fetch SSL certificate
-    config = CrawlerRunConfig(
-        fetch_ssl_certificate=True,
-        cache_mode=CacheMode.BYPASS,  # Bypass cache to always get fresh certificates
-    )
-
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun(url="https://example.com", config=config)
-
-        if result.success and result.ssl_certificate:
-            cert = result.ssl_certificate
-
-            tmp_dir = os.path.join(__location__, "tmp")
-            os.makedirs(tmp_dir, exist_ok=True)
-
-            # 1. Access certificate properties directly
-            print("\nCertificate Information:")
-            print(f"Issuer: {cert.issuer.get('CN', '')}")
-            print(f"Valid until: {cert.valid_until}")
-            print(f"Fingerprint: {cert.fingerprint}")
-
-            # 2. Export certificate in different formats
-            cert.to_json(os.path.join(tmp_dir, "certificate.json"))  # For analysis
-            print("\nCertificate exported to:")
-            print(f"- JSON: {os.path.join(tmp_dir, 'certificate.json')}")
-
-            pem_data = cert.to_pem(
-                os.path.join(tmp_dir, "certificate.pem")
-            )  # For web servers
-            print(f"- PEM: {os.path.join(tmp_dir, 'certificate.pem')}")
-
-            der_data = cert.to_der(
-                os.path.join(tmp_dir, "certificate.der")
-            )  # For Java apps
-            print(f"- DER: {os.path.join(tmp_dir, 'certificate.der')}")
-
-
-# Main execution
-async def main():
-    # Basic examples
-    await simple_crawl()
-    await simple_example_with_running_js_code()
-    await simple_example_with_css_selector()
-
-    # Advanced examples
-    await extract_structured_data_using_css_extractor()
-    await extract_structured_data_using_llm(
-        "openai/gpt-4o", os.getenv("OPENAI_API_KEY")
-    )
-    await crawl_dynamic_content_pages_method_1()
-    await crawl_dynamic_content_pages_method_2()
-
-    # Browser comparisons
-    await crawl_custom_browser_type()
-
-    # Screenshot example
-    await capture_and_save_screenshot(
-        "https://www.example.com",
-        os.path.join(__location__, "tmp/example_screenshot.jpg")
-    )
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

docs/examples/quickstart_sync.py

@@ -0,0 +1,405 @@
+import os
+import time
+from crawl4ai import LLMConfig
+from crawl4ai.web_crawler import WebCrawler
+from crawl4ai.chunking_strategy import *
+from crawl4ai.extraction_strategy import *
+from crawl4ai.crawler_strategy import *
+from rich import print
+from rich.console import Console
+from functools import lru_cache
+
+console = Console()
+
+
+@lru_cache()
+def create_crawler():
+    crawler = WebCrawler(verbose=True)
+    crawler.warmup()
+    return crawler
+
+
+def print_result(result):
+    # Print each key in one line and just the first 10 characters of each one's value and three dots
+    console.print("\t[bold]Result:[/bold]")
+    for key, value in result.model_dump().items():
+        if isinstance(value, str) and value:
+            console.print(f"\t{key}: [green]{value[:20]}...[/green]")
+    if result.extracted_content:
+        items = json.loads(result.extracted_content)
+        print(f"\t[bold]{len(items)} blocks is extracted![/bold]")
+
+
+def cprint(message, press_any_key=False):
+    console.print(message)
+    if press_any_key:
+        console.print("Press any key to continue...", style="")
+        input()
+
+
+def basic_usage(crawler):
+    cprint(
+        "🛠️ [bold cyan]Basic Usage: Simply provide a URL and let Crawl4ai do the magic![/bold cyan]"
+    )
+    result = crawler.run(url="https://www.nbcnews.com/business", only_text=True)
+    cprint("[LOG] 📦 [bold yellow]Basic crawl result:[/bold yellow]")
+    print_result(result)
+
+
+def basic_usage_some_params(crawler):
+    cprint(
+        "🛠️ [bold cyan]Basic Usage: Simply provide a URL and let Crawl4ai do the magic![/bold cyan]"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business", word_count_threshold=1, only_text=True
+    )
+    cprint("[LOG] 📦 [bold yellow]Basic crawl result:[/bold yellow]")
+    print_result(result)
+
+
+def screenshot_usage(crawler):
+    cprint("\n📸 [bold cyan]Let's take a screenshot of the page![/bold cyan]")
+    result = crawler.run(url="https://www.nbcnews.com/business", screenshot=True)
+    cprint("[LOG] 📦 [bold yellow]Screenshot result:[/bold yellow]")
+    # Save the screenshot to a file
+    with open("screenshot.png", "wb") as f:
+        f.write(base64.b64decode(result.screenshot))
+    cprint("Screenshot saved to 'screenshot.png'!")
+    print_result(result)
+
+
+def understanding_parameters(crawler):
+    cprint(
+        "\n🧠 [bold cyan]Understanding 'bypass_cache' and 'include_raw_html' parameters:[/bold cyan]"
+    )
+    cprint(
+        "By default, Crawl4ai caches the results of your crawls. This means that subsequent crawls of the same URL will be much faster! Let's see this in action."
+    )
+
+    # First crawl (reads from cache)
+    cprint("1️⃣ First crawl (caches the result):", True)
+    start_time = time.time()
+    result = crawler.run(url="https://www.nbcnews.com/business")
+    end_time = time.time()
+    cprint(
+        f"[LOG] 📦 [bold yellow]First crawl took {end_time - start_time} seconds and result (from cache):[/bold yellow]"
+    )
+    print_result(result)
+
+    # Force to crawl again
+    cprint("2️⃣ Second crawl (Force to crawl again):", True)
+    start_time = time.time()
+    result = crawler.run(url="https://www.nbcnews.com/business", bypass_cache=True)
+    end_time = time.time()
+    cprint(
+        f"[LOG] 📦 [bold yellow]Second crawl took {end_time - start_time} seconds and result (forced to crawl):[/bold yellow]"
+    )
+    print_result(result)
+
+
+def add_chunking_strategy(crawler):
+    # Adding a chunking strategy: RegexChunking
+    cprint(
+        "\n🧩 [bold cyan]Let's add a chunking strategy: RegexChunking![/bold cyan]",
+        True,
+    )
+    cprint(
+        "RegexChunking is a simple chunking strategy that splits the text based on a given regex pattern. Let's see it in action!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        chunking_strategy=RegexChunking(patterns=["\n\n"]),
+    )
+    cprint("[LOG] 📦 [bold yellow]RegexChunking result:[/bold yellow]")
+    print_result(result)
+
+    # Adding another chunking strategy: NlpSentenceChunking
+    cprint(
+        "\n🔍 [bold cyan]Time to explore another chunking strategy: NlpSentenceChunking![/bold cyan]",
+        True,
+    )
+    cprint(
+        "NlpSentenceChunking uses NLP techniques to split the text into sentences. Let's see how it performs!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business", chunking_strategy=NlpSentenceChunking()
+    )
+    cprint("[LOG] 📦 [bold yellow]NlpSentenceChunking result:[/bold yellow]")
+    print_result(result)
+
+
+def add_extraction_strategy(crawler):
+    # Adding an extraction strategy: CosineStrategy
+    cprint(
+        "\n🧠 [bold cyan]Let's get smarter with an extraction strategy: CosineStrategy![/bold cyan]",
+        True,
+    )
+    cprint(
+        "CosineStrategy uses cosine similarity to extract semantically similar blocks of text. Let's see it in action!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=CosineStrategy(
+            word_count_threshold=10,
+            max_dist=0.2,
+            linkage_method="ward",
+            top_k=3,
+            sim_threshold=0.3,
+            verbose=True,
+        ),
+    )
+    cprint("[LOG] 📦 [bold yellow]CosineStrategy result:[/bold yellow]")
+    print_result(result)
+
+    # Using semantic_filter with CosineStrategy
+    cprint(
+        "You can pass other parameters like 'semantic_filter' to the CosineStrategy to extract semantically similar blocks of text. Let's see it in action!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=CosineStrategy(
+            semantic_filter="inflation rent prices",
+        ),
+    )
+    cprint(
+        "[LOG] 📦 [bold yellow]CosineStrategy result with semantic filter:[/bold yellow]"
+    )
+    print_result(result)
+
+
+def add_llm_extraction_strategy(crawler):
+    # Adding an LLM extraction strategy without instructions
+    cprint(
+        "\n🤖 [bold cyan]Time to bring in the big guns: LLMExtractionStrategy without instructions![/bold cyan]",
+        True,
+    )
+    cprint(
+        "LLMExtractionStrategy uses a large language model to extract relevant information from the web page. Let's see it in action!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config =  LLMConfig(provider="openai/gpt-4o", api_token=os.getenv("OPENAI_API_KEY"))
+        ),
+    )
+    cprint(
+        "[LOG] 📦 [bold yellow]LLMExtractionStrategy (no instructions) result:[/bold yellow]"
+    )
+    print_result(result)
+
+    # Adding an LLM extraction strategy with instructions
+    cprint(
+        "\n📜 [bold cyan]Let's make it even more interesting: LLMExtractionStrategy with instructions![/bold cyan]",
+        True,
+    )
+    cprint(
+        "Let's say we are only interested in financial news. Let's see how LLMExtractionStrategy performs with instructions!"
+    )
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider="openai/gpt-4o",api_token=os.getenv("OPENAI_API_KEY")),
+            instruction="I am interested in only financial news",
+        ),
+    )
+    cprint(
+        "[LOG] 📦 [bold yellow]LLMExtractionStrategy (with instructions) result:[/bold yellow]"
+    )
+    print_result(result)
+
+    result = crawler.run(
+        url="https://www.nbcnews.com/business",
+        extraction_strategy=LLMExtractionStrategy(
+            llm_config=LLMConfig(provider="openai/gpt-4o",api_token=os.getenv("OPENAI_API_KEY")),
+            instruction="Extract only content related to technology",
+        ),
+    )
+    cprint(
+        "[LOG] 📦 [bold yellow]LLMExtractionStrategy (with technology instruction) result:[/bold yellow]"
+    )
+    print_result(result)
+
+
+def targeted_extraction(crawler):
+    # Using a CSS selector to extract only H2 tags
+    cprint(
+        "\n🎯 [bold cyan]Targeted extraction: Let's use a CSS selector to extract only H2 tags![/bold cyan]",
+        True,
+    )
+    result = crawler.run(url="https://www.nbcnews.com/business", css_selector="h2")
+    cprint("[LOG] 📦 [bold yellow]CSS Selector (H2 tags) result:[/bold yellow]")
+    print_result(result)
+
+
+def interactive_extraction(crawler):
+    # Passing JavaScript code to interact with the page
+    cprint(
+        "\n🖱️ [bold cyan]Let's get interactive: Passing JavaScript code to click 'Load More' button![/bold cyan]",
+        True,
+    )
+    cprint(
+        "In this example we try to click the 'Load More' button on the page using JavaScript code."
+    )
+    js_code = """
+    const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More'));
+    loadMoreButton && loadMoreButton.click();
+    """
+    # crawler_strategy = LocalSeleniumCrawlerStrategy(js_code=js_code)
+    # crawler = WebCrawler(crawler_strategy=crawler_strategy, always_by_pass_cache=True)
+    result = crawler.run(url="https://www.nbcnews.com/business", js=js_code)
+    cprint(
+        "[LOG] 📦 [bold yellow]JavaScript Code (Load More button) result:[/bold yellow]"
+    )
+    print_result(result)
+
+
+def multiple_scrip(crawler):
+    # Passing JavaScript code to interact with the page
+    cprint(
+        "\n🖱️ [bold cyan]Let's get interactive: Passing JavaScript code to click 'Load More' button![/bold cyan]",
+        True,
+    )
+    cprint(
+        "In this example we try to click the 'Load More' button on the page using JavaScript code."
+    )
+    js_code = [
+        """
+    const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More'));
+    loadMoreButton && loadMoreButton.click();
+    """
+    ] * 2
+    # crawler_strategy = LocalSeleniumCrawlerStrategy(js_code=js_code)
+    # crawler = WebCrawler(crawler_strategy=crawler_strategy, always_by_pass_cache=True)
+    result = crawler.run(url="https://www.nbcnews.com/business", js=js_code)
+    cprint(
+        "[LOG] 📦 [bold yellow]JavaScript Code (Load More button) result:[/bold yellow]"
+    )
+    print_result(result)
+
+
+def using_crawler_hooks(crawler):
+    # Example usage of the hooks for authentication and setting a cookie
+    def on_driver_created(driver):
+        print("[HOOK] on_driver_created")
+        # Example customization: maximize the window
+        driver.maximize_window()
+
+        # Example customization: logging in to a hypothetical website
+        driver.get("https://example.com/login")
+
+        from selenium.webdriver.support.ui import WebDriverWait
+        from selenium.webdriver.common.by import By
+        from selenium.webdriver.support import expected_conditions as EC
+
+        WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.NAME, "username"))
+        )
+        driver.find_element(By.NAME, "username").send_keys("testuser")
+        driver.find_element(By.NAME, "password").send_keys("password123")
+        driver.find_element(By.NAME, "login").click()
+        WebDriverWait(driver, 10).until(
+            EC.presence_of_element_located((By.ID, "welcome"))
+        )
+        # Add a custom cookie
+        driver.add_cookie({"name": "test_cookie", "value": "cookie_value"})
+        return driver
+
+    def before_get_url(driver):
+        print("[HOOK] before_get_url")
+        # Example customization: add a custom header
+        # Enable Network domain for sending headers
+        driver.execute_cdp_cmd("Network.enable", {})
+        # Add a custom header
+        driver.execute_cdp_cmd(
+            "Network.setExtraHTTPHeaders", {"headers": {"X-Test-Header": "test"}}
+        )
+        return driver
+
+    def after_get_url(driver):
+        print("[HOOK] after_get_url")
+        # Example customization: log the URL
+        print(driver.current_url)
+        return driver
+
+    def before_return_html(driver, html):
+        print("[HOOK] before_return_html")
+        # Example customization: log the HTML
+        print(len(html))
+        return driver
+
+    cprint(
+        "\n🔗 [bold cyan]Using Crawler Hooks: Let's see how we can customize the crawler using hooks![/bold cyan]",
+        True,
+    )
+
+    crawler_strategy = LocalSeleniumCrawlerStrategy(verbose=True)
+    crawler_strategy.set_hook("on_driver_created", on_driver_created)
+    crawler_strategy.set_hook("before_get_url", before_get_url)
+    crawler_strategy.set_hook("after_get_url", after_get_url)
+    crawler_strategy.set_hook("before_return_html", before_return_html)
+
+    crawler = WebCrawler(verbose=True, crawler_strategy=crawler_strategy)
+    crawler.warmup()
+    result = crawler.run(url="https://example.com")
+
+    cprint("[LOG] 📦 [bold yellow]Crawler Hooks result:[/bold yellow]")
+    print_result(result=result)
+
+
+def using_crawler_hooks_dleay_example(crawler):
+    def delay(driver):
+        print("Delaying for 5 seconds...")
+        time.sleep(5)
+        print("Resuming...")
+
+    def create_crawler():
+        crawler_strategy = LocalSeleniumCrawlerStrategy(verbose=True)
+        crawler_strategy.set_hook("after_get_url", delay)
+        crawler = WebCrawler(verbose=True, crawler_strategy=crawler_strategy)
+        crawler.warmup()
+        return crawler
+
+    cprint(
+        "\n🔗 [bold cyan]Using Crawler Hooks: Let's add a delay after fetching the url to make sure entire page is fetched.[/bold cyan]"
+    )
+    crawler = create_crawler()
+    result = crawler.run(url="https://google.com", bypass_cache=True)
+
+    cprint("[LOG] 📦 [bold yellow]Crawler Hooks result:[/bold yellow]")
+    print_result(result)
+
+
+def main():
+    cprint(
+        "🌟 [bold green]Welcome to the Crawl4ai Quickstart Guide! Let's dive into some web crawling fun! 🌐[/bold green]"
+    )
+    cprint(
+        "⛳️ [bold cyan]First Step: Create an instance of WebCrawler and call the `warmup()` function.[/bold cyan]"
+    )
+    cprint(
+        "If this is the first time you're running Crawl4ai, this might take a few seconds to load required model files."
+    )
+
+    crawler = create_crawler()
+
+    crawler.always_by_pass_cache = True
+    basic_usage(crawler)
+    # basic_usage_some_params(crawler)
+    understanding_parameters(crawler)
+
+    crawler.always_by_pass_cache = True
+    screenshot_usage(crawler)
+    add_chunking_strategy(crawler)
+    add_extraction_strategy(crawler)
+    add_llm_extraction_strategy(crawler)
+    targeted_extraction(crawler)
+    interactive_extraction(crawler)
+    multiple_scrip(crawler)
+
+    cprint(
+        "\n🎉 [bold green]Congratulations! You've made it through the Crawl4ai Quickstart Guide! Now go forth and crawl the web like a pro! 🕸️[/bold green]"
+    )
+
+
+if __name__ == "__main__":
+    main()

docs/examples/quickstart_v0.ipynb

@@ -0,0 +1,735 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6yLvrXn7yZQI"
+      },
+      "source": [
+        "# Crawl4AI: Advanced Web Crawling and Data Extraction\n",
+        "\n",
+        "Welcome to this interactive notebook showcasing Crawl4AI, an advanced asynchronous web crawling and data extraction library.\n",
+        "\n",
+        "- GitHub Repository: [https://github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)\n",
+        "- Twitter: [@unclecode](https://twitter.com/unclecode)\n",
+        "- Website: [https://crawl4ai.com](https://crawl4ai.com)\n",
+        "\n",
+        "Let's explore the powerful features of Crawl4AI!"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KIn_9nxFyZQK"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "First, let's install Crawl4AI from GitHub:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mSnaxLf3zMog"
+      },
+      "outputs": [],
+      "source": [
+        "!sudo apt-get update && sudo apt-get install -y libwoff1 libopus0 libwebp6 libwebpdemux2 libenchant1c2a libgudev-1.0-0 libsecret-1-0 libhyphen0 libgdk-pixbuf2.0-0 libegl1 libnotify4 libxslt1.1 libevent-2.1-7 libgles2 libvpx6 libxcomposite1 libatk1.0-0 libatk-bridge2.0-0 libepoxy0 libgtk-3-0 libharfbuzz-icu0"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xlXqaRtayZQK"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install crawl4ai\n",
+        "!pip install nest-asyncio\n",
+        "!playwright install"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qKCE7TI7yZQL"
+      },
+      "source": [
+        "Now, let's import the necessary libraries:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 1,
+      "metadata": {
+        "id": "I67tr7aAyZQL"
+      },
+      "outputs": [],
+      "source": [
+        "import asyncio\n",
+        "import nest_asyncio\n",
+        "from crawl4ai import AsyncWebCrawler\n",
+        "from crawl4ai.extraction_strategy import JsonCssExtractionStrategy, LLMExtractionStrategy\n",
+        "import json\n",
+        "import time\n",
+        "from pydantic import BaseModel, Field\n",
+        "\n",
+        "nest_asyncio.apply()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h7yR_Rt_yZQM"
+      },
+      "source": [
+        "## Basic Usage\n",
+        "\n",
+        "Let's start with a simple crawl example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "yBh6hf4WyZQM",
+        "outputId": "0f83af5c-abba-4175-ed95-70b7512e6bcc"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.05 seconds\n",
+            "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.05 seconds.\n",
+            "18102\n"
+          ]
+        }
+      ],
+      "source": [
+        "async def simple_crawl():\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        result = await crawler.arun(url=\"https://www.nbcnews.com/business\")\n",
+        "        print(len(result.markdown))\n",
+        "await simple_crawl()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9rtkgHI28uI4"
+      },
+      "source": [
+        "💡 By default, **Crawl4AI** caches the result of every URL, so the next time you call it, you’ll get an instant result. But if you want to bypass the cache, just set `bypass_cache=True`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MzZ0zlJ9yZQM"
+      },
+      "source": [
+        "## Advanced Features\n",
+        "\n",
+        "### Executing JavaScript and Using CSS Selectors"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "gHStF86xyZQM",
+        "outputId": "34d0fb6d-4dec-4677-f76e-85a1f082829b"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🕸️ Crawling https://www.nbcnews.com/business using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://www.nbcnews.com/business successfully!\n",
+            "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business, success: True, time taken: 6.06 seconds\n",
+            "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.10 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.11 seconds.\n",
+            "41135\n"
+          ]
+        }
+      ],
+      "source": [
+        "async def js_and_css():\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        js_code = [\"const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();\"]\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            js_code=js_code,\n",
+        "            # css_selector=\"YOUR_CSS_SELECTOR_HERE\",\n",
+        "            bypass_cache=True\n",
+        "        )\n",
+        "        print(len(result.markdown))\n",
+        "\n",
+        "await js_and_css()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cqE_W4coyZQM"
+      },
+      "source": [
+        "### Using a Proxy\n",
+        "\n",
+        "Note: You'll need to replace the proxy URL with a working proxy for this example to run successfully."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QjAyiAGqyZQM"
+      },
+      "outputs": [],
+      "source": [
+        "async def use_proxy():\n",
+        "    async with AsyncWebCrawler(verbose=True, proxy=\"http://your-proxy-url:port\") as crawler:\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            bypass_cache=True\n",
+        "        )\n",
+        "        print(result.markdown[:500])  # Print first 500 characters\n",
+        "\n",
+        "# Uncomment the following line to run the proxy example\n",
+        "# await use_proxy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XTZ88lbayZQN"
+      },
+      "source": [
+        "### Extracting Structured Data with OpenAI\n",
+        "\n",
+        "Note: You'll need to set your OpenAI API key as an environment variable for this example to work."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 14,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "fIOlDayYyZQN",
+        "outputId": "cb8359cc-dee0-4762-9698-5dfdcee055b8"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🕸️ Crawling https://openai.com/api/pricing/ using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://openai.com/api/pricing/ successfully!\n",
+            "[LOG] 🚀 Crawling done for https://openai.com/api/pricing/, success: True, time taken: 3.77 seconds\n",
+            "[LOG] 🚀 Content extracted for https://openai.com/api/pricing/, success: True, time taken: 0.21 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://openai.com/api/pricing/, Strategy: AsyncWebCrawler\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 0\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 1\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 2\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 3\n",
+            "[LOG] Extracted 4 blocks from URL: https://openai.com/api/pricing/ block index: 3\n",
+            "[LOG] Call LLM for https://openai.com/api/pricing/ - block index: 4\n",
+            "[LOG] Extracted 5 blocks from URL: https://openai.com/api/pricing/ block index: 0\n",
+            "[LOG] Extracted 1 blocks from URL: https://openai.com/api/pricing/ block index: 4\n",
+            "[LOG] Extracted 8 blocks from URL: https://openai.com/api/pricing/ block index: 1\n",
+            "[LOG] Extracted 12 blocks from URL: https://openai.com/api/pricing/ block index: 2\n",
+            "[LOG] 🚀 Extraction done for https://openai.com/api/pricing/, time taken: 8.55 seconds.\n",
+            "5029\n"
+          ]
+        }
+      ],
+      "source": [
+        "import os\n",
+        "from google.colab import userdata\n",
+        "os.environ['OPENAI_API_KEY'] = userdata.get('OPENAI_API_KEY')\n",
+        "\n",
+        "class OpenAIModelFee(BaseModel):\n",
+        "    model_name: str = Field(..., description=\"Name of the OpenAI model.\")\n",
+        "    input_fee: str = Field(..., description=\"Fee for input token for the OpenAI model.\")\n",
+        "    output_fee: str = Field(..., description=\"Fee for output token for the OpenAI model.\")\n",
+        "\n",
+        "async def extract_openai_fees():\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        result = await crawler.arun(\n",
+        "            url='https://openai.com/api/pricing/',\n",
+        "            word_count_threshold=1,\n",
+        "            extraction_strategy=LLMExtractionStrategy(\n",
+        "                provider=\"openai/gpt-4o\", api_token=os.getenv('OPENAI_API_KEY'),\n",
+        "                schema=OpenAIModelFee.schema(),\n",
+        "                extraction_type=\"schema\",\n",
+        "                instruction=\"\"\"From the crawled content, extract all mentioned model names along with their fees for input and output tokens.\n",
+        "                Do not miss any models in the entire content. One extracted model JSON format should look like this:\n",
+        "                {\"model_name\": \"GPT-4\", \"input_fee\": \"US$10.00 / 1M tokens\", \"output_fee\": \"US$30.00 / 1M tokens\"}.\"\"\"\n",
+        "            ),\n",
+        "            bypass_cache=True,\n",
+        "        )\n",
+        "        print(len(result.extracted_content))\n",
+        "\n",
+        "# Uncomment the following line to run the OpenAI extraction example\n",
+        "await extract_openai_fees()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BypA5YxEyZQN"
+      },
+      "source": [
+        "### Advanced Multi-Page Crawling with JavaScript Execution"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tfkcVQ0b7mw-"
+      },
+      "source": [
+        "## Advanced Multi-Page Crawling with JavaScript Execution\n",
+        "\n",
+        "This example demonstrates Crawl4AI's ability to handle complex crawling scenarios, specifically extracting commits from multiple pages of a GitHub repository. The challenge here is that clicking the \"Next\" button doesn't load a new page, but instead uses asynchronous JavaScript to update the content. This is a common hurdle in modern web crawling.\n",
+        "\n",
+        "To overcome this, we use Crawl4AI's custom JavaScript execution to simulate clicking the \"Next\" button, and implement a custom hook to detect when new data has loaded. Our strategy involves comparing the first commit's text before and after \"clicking\" Next, waiting until it changes to confirm new data has rendered. This showcases Crawl4AI's flexibility in handling dynamic content and its ability to implement custom logic for even the most challenging crawling tasks."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "qUBKGpn3yZQN",
+        "outputId": "3e555b6a-ed33-42f4-cce9-499a923fbe17"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🕸️ Crawling https://github.com/microsoft/TypeScript/commits/main using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://github.com/microsoft/TypeScript/commits/main successfully!\n",
+            "[LOG] 🚀 Crawling done for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 5.16 seconds\n",
+            "[LOG] 🚀 Content extracted for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 0.28 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://github.com/microsoft/TypeScript/commits/main, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://github.com/microsoft/TypeScript/commits/main, time taken: 0.28 seconds.\n",
+            "Page 1: Found 35 commits\n",
+            "[LOG] 🕸️ Crawling https://github.com/microsoft/TypeScript/commits/main using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://github.com/microsoft/TypeScript/commits/main successfully!\n",
+            "[LOG] 🚀 Crawling done for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 0.78 seconds\n",
+            "[LOG] 🚀 Content extracted for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 0.90 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://github.com/microsoft/TypeScript/commits/main, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://github.com/microsoft/TypeScript/commits/main, time taken: 0.90 seconds.\n",
+            "Page 2: Found 35 commits\n",
+            "[LOG] 🕸️ Crawling https://github.com/microsoft/TypeScript/commits/main using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://github.com/microsoft/TypeScript/commits/main successfully!\n",
+            "[LOG] 🚀 Crawling done for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 2.00 seconds\n",
+            "[LOG] 🚀 Content extracted for https://github.com/microsoft/TypeScript/commits/main, success: True, time taken: 0.74 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://github.com/microsoft/TypeScript/commits/main, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://github.com/microsoft/TypeScript/commits/main, time taken: 0.75 seconds.\n",
+            "Page 3: Found 35 commits\n",
+            "Successfully crawled 105 commits across 3 pages\n"
+          ]
+        }
+      ],
+      "source": [
+        "import re\n",
+        "from bs4 import BeautifulSoup\n",
+        "\n",
+        "async def crawl_typescript_commits():\n",
+        "    first_commit = \"\"\n",
+        "    async def on_execution_started(page):\n",
+        "        nonlocal first_commit\n",
+        "        try:\n",
+        "            while True:\n",
+        "                await page.wait_for_selector('li.Box-sc-g0xbh4-0 h4')\n",
+        "                commit = await page.query_selector('li.Box-sc-g0xbh4-0 h4')\n",
+        "                commit = await commit.evaluate('(element) => element.textContent')\n",
+        "                commit = re.sub(r'\\s+', '', commit)\n",
+        "                if commit and commit != first_commit:\n",
+        "                    first_commit = commit\n",
+        "                    break\n",
+        "                await asyncio.sleep(0.5)\n",
+        "        except Exception as e:\n",
+        "            print(f\"Warning: New content didn't appear after JavaScript execution: {e}\")\n",
+        "\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        crawler.crawler_strategy.set_hook('on_execution_started', on_execution_started)\n",
+        "\n",
+        "        url = \"https://github.com/microsoft/TypeScript/commits/main\"\n",
+        "        session_id = \"typescript_commits_session\"\n",
+        "        all_commits = []\n",
+        "\n",
+        "        js_next_page = \"\"\"\n",
+        "        const button = document.querySelector('a[data-testid=\"pagination-next-button\"]');\n",
+        "        if (button) button.click();\n",
+        "        \"\"\"\n",
+        "\n",
+        "        for page in range(3):  # Crawl 3 pages\n",
+        "            result = await crawler.arun(\n",
+        "                url=url,\n",
+        "                session_id=session_id,\n",
+        "                css_selector=\"li.Box-sc-g0xbh4-0\",\n",
+        "                js=js_next_page if page > 0 else None,\n",
+        "                bypass_cache=True,\n",
+        "                js_only=page > 0\n",
+        "            )\n",
+        "\n",
+        "            assert result.success, f\"Failed to crawl page {page + 1}\"\n",
+        "\n",
+        "            soup = BeautifulSoup(result.cleaned_html, 'html.parser')\n",
+        "            commits = soup.select(\"li\")\n",
+        "            all_commits.extend(commits)\n",
+        "\n",
+        "            print(f\"Page {page + 1}: Found {len(commits)} commits\")\n",
+        "\n",
+        "        await crawler.crawler_strategy.kill_session(session_id)\n",
+        "        print(f\"Successfully crawled {len(all_commits)} commits across 3 pages\")\n",
+        "\n",
+        "await crawl_typescript_commits()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EJRnYsp6yZQN"
+      },
+      "source": [
+        "### Using JsonCssExtractionStrategy for Fast Structured Output"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1ZMqIzB_8SYp"
+      },
+      "source": [
+        "The JsonCssExtractionStrategy is a powerful feature of Crawl4AI that allows for precise, structured data extraction from web pages. Here's how it works:\n",
+        "\n",
+        "1. You define a schema that describes the pattern of data you're interested in extracting.\n",
+        "2. The schema includes a base selector that identifies repeating elements on the page.\n",
+        "3. Within the schema, you define fields, each with its own selector and type.\n",
+        "4. These field selectors are applied within the context of each base selector element.\n",
+        "5. The strategy supports nested structures, lists within lists, and various data types.\n",
+        "6. You can even include computed fields for more complex data manipulation.\n",
+        "\n",
+        "This approach allows for highly flexible and precise data extraction, transforming semi-structured web content into clean, structured JSON data. It's particularly useful for extracting consistent data patterns from pages like product listings, news articles, or search results.\n",
+        "\n",
+        "For more details and advanced usage, check out the full documentation on the Crawl4AI website."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 12,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "trCMR2T9yZQN",
+        "outputId": "718d36f4-cccf-40f4-8d8c-c3ba73524d16"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[LOG] 🌤️  Warming up the AsyncWebCrawler\n",
+            "[LOG] 🌞 AsyncWebCrawler is ready to crawl\n",
+            "[LOG] 🕸️ Crawling https://www.nbcnews.com/business using AsyncPlaywrightCrawlerStrategy...\n",
+            "[LOG] ✅ Crawled https://www.nbcnews.com/business successfully!\n",
+            "[LOG] 🚀 Crawling done for https://www.nbcnews.com/business, success: True, time taken: 7.00 seconds\n",
+            "[LOG] 🚀 Content extracted for https://www.nbcnews.com/business, success: True, time taken: 0.32 seconds\n",
+            "[LOG] 🔥 Extracting semantic blocks for https://www.nbcnews.com/business, Strategy: AsyncWebCrawler\n",
+            "[LOG] 🚀 Extraction done for https://www.nbcnews.com/business, time taken: 0.48 seconds.\n",
+            "Successfully extracted 11 news teasers\n",
+            "{\n",
+            "  \"category\": \"Business News\",\n",
+            "  \"headline\": \"NBC ripped up its Olympics playbook for 2024 \\u2014 so far, the new strategy paid off\",\n",
+            "  \"summary\": \"The Olympics have long been key to NBCUniversal. Paris marked the 18th Olympic Games broadcast by NBC in the U.S.\",\n",
+            "  \"time\": \"13h ago\",\n",
+            "  \"image\": {\n",
+            "    \"src\": \"https://media-cldnry.s-nbcnews.com/image/upload/t_focal-200x100,f_auto,q_auto:best/rockcms/2024-09/240903-nbc-olympics-ch-1344-c7a486.jpg\",\n",
+            "    \"alt\": \"Mike Tirico.\"\n",
+            "  },\n",
+            "  \"link\": \"https://www.nbcnews.com/business\"\n",
+            "}\n"
+          ]
+        }
+      ],
+      "source": [
+        "async def extract_news_teasers():\n",
+        "    schema = {\n",
+        "        \"name\": \"News Teaser Extractor\",\n",
+        "        \"baseSelector\": \".wide-tease-item__wrapper\",\n",
+        "        \"fields\": [\n",
+        "            {\n",
+        "                \"name\": \"category\",\n",
+        "                \"selector\": \".unibrow span[data-testid='unibrow-text']\",\n",
+        "                \"type\": \"text\",\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"headline\",\n",
+        "                \"selector\": \".wide-tease-item__headline\",\n",
+        "                \"type\": \"text\",\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"summary\",\n",
+        "                \"selector\": \".wide-tease-item__description\",\n",
+        "                \"type\": \"text\",\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"time\",\n",
+        "                \"selector\": \"[data-testid='wide-tease-date']\",\n",
+        "                \"type\": \"text\",\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"image\",\n",
+        "                \"type\": \"nested\",\n",
+        "                \"selector\": \"picture.teasePicture img\",\n",
+        "                \"fields\": [\n",
+        "                    {\"name\": \"src\", \"type\": \"attribute\", \"attribute\": \"src\"},\n",
+        "                    {\"name\": \"alt\", \"type\": \"attribute\", \"attribute\": \"alt\"},\n",
+        "                ],\n",
+        "            },\n",
+        "            {\n",
+        "                \"name\": \"link\",\n",
+        "                \"selector\": \"a[href]\",\n",
+        "                \"type\": \"attribute\",\n",
+        "                \"attribute\": \"href\",\n",
+        "            },\n",
+        "        ],\n",
+        "    }\n",
+        "\n",
+        "    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)\n",
+        "\n",
+        "    async with AsyncWebCrawler(verbose=True) as crawler:\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            extraction_strategy=extraction_strategy,\n",
+        "            bypass_cache=True,\n",
+        "        )\n",
+        "\n",
+        "        assert result.success, \"Failed to crawl the page\"\n",
+        "\n",
+        "        news_teasers = json.loads(result.extracted_content)\n",
+        "        print(f\"Successfully extracted {len(news_teasers)} news teasers\")\n",
+        "        print(json.dumps(news_teasers[0], indent=2))\n",
+        "\n",
+        "await extract_news_teasers()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FnyVhJaByZQN"
+      },
+      "source": [
+        "## Speed Comparison\n",
+        "\n",
+        "Let's compare the speed of Crawl4AI with Firecrawl, a paid service. Note that we can't run Firecrawl in this Colab environment, so we'll simulate its performance based on previously recorded data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "agDD186f3wig"
+      },
+      "source": [
+        "💡 **Note on Speed Comparison:**\n",
+        "\n",
+        "The speed test conducted here is running on Google Colab, where the internet speed and performance can vary and may not reflect optimal conditions. When we call Firecrawl's API, we're seeing its best performance, while Crawl4AI's performance is limited by Colab's network speed.\n",
+        "\n",
+        "For a more accurate comparison, it's recommended to run these tests on your own servers or computers with a stable and fast internet connection. Despite these limitations, Crawl4AI still demonstrates faster performance in this environment.\n",
+        "\n",
+        "If you run these tests locally, you may observe an even more significant speed advantage for Crawl4AI compared to other services."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "F7KwHv8G1LbY"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install firecrawl"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "91813zILyZQN",
+        "outputId": "663223db-ab89-4976-b233-05ceca62b19b"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Firecrawl (simulated):\n",
+            "Time taken: 4.38 seconds\n",
+            "Content length: 41967 characters\n",
+            "Images found: 49\n",
+            "\n",
+            "Crawl4AI (simple crawl):\n",
+            "Time taken: 4.22 seconds\n",
+            "Content length: 18221 characters\n",
+            "Images found: 49\n",
+            "\n",
+            "Crawl4AI (with JavaScript execution):\n",
+            "Time taken: 9.13 seconds\n",
+            "Content length: 34243 characters\n",
+            "Images found: 89\n"
+          ]
+        }
+      ],
+      "source": [
+        "import os\n",
+        "from google.colab import userdata\n",
+        "os.environ['FIRECRAWL_API_KEY'] = userdata.get('FIRECRAWL_API_KEY')\n",
+        "import time\n",
+        "from firecrawl import FirecrawlApp\n",
+        "\n",
+        "async def speed_comparison():\n",
+        "    # Simulated Firecrawl performance\n",
+        "    app = FirecrawlApp(api_key=os.environ['FIRECRAWL_API_KEY'])\n",
+        "    start = time.time()\n",
+        "    scrape_status = app.scrape_url(\n",
+        "    'https://www.nbcnews.com/business',\n",
+        "    params={'formats': ['markdown', 'html']}\n",
+        "    )\n",
+        "    end = time.time()\n",
+        "    print(\"Firecrawl (simulated):\")\n",
+        "    print(f\"Time taken: {end - start:.2f} seconds\")\n",
+        "    print(f\"Content length: {len(scrape_status['markdown'])} characters\")\n",
+        "    print(f\"Images found: {scrape_status['markdown'].count('cldnry.s-nbcnews.com')}\")\n",
+        "    print()\n",
+        "\n",
+        "    async with AsyncWebCrawler() as crawler:\n",
+        "        # Crawl4AI simple crawl\n",
+        "        start = time.time()\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            word_count_threshold=0,\n",
+        "            bypass_cache=True,\n",
+        "            verbose=False\n",
+        "        )\n",
+        "        end = time.time()\n",
+        "        print(\"Crawl4AI (simple crawl):\")\n",
+        "        print(f\"Time taken: {end - start:.2f} seconds\")\n",
+        "        print(f\"Content length: {len(result.markdown)} characters\")\n",
+        "        print(f\"Images found: {result.markdown.count('cldnry.s-nbcnews.com')}\")\n",
+        "        print()\n",
+        "\n",
+        "        # Crawl4AI with JavaScript execution\n",
+        "        start = time.time()\n",
+        "        result = await crawler.arun(\n",
+        "            url=\"https://www.nbcnews.com/business\",\n",
+        "            js_code=[\"const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();\"],\n",
+        "            word_count_threshold=0,\n",
+        "            bypass_cache=True,\n",
+        "            verbose=False\n",
+        "        )\n",
+        "        end = time.time()\n",
+        "        print(\"Crawl4AI (with JavaScript execution):\")\n",
+        "        print(f\"Time taken: {end - start:.2f} seconds\")\n",
+        "        print(f\"Content length: {len(result.markdown)} characters\")\n",
+        "        print(f\"Images found: {result.markdown.count('cldnry.s-nbcnews.com')}\")\n",
+        "\n",
+        "await speed_comparison()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OBFFYVJIyZQN"
+      },
+      "source": [
+        "If you run on a local machine with a proper internet speed:\n",
+        "- Simple crawl: Crawl4AI is typically over 3-4 times faster than Firecrawl.\n",
+        "- With JavaScript execution: Even when executing JavaScript to load more content (potentially doubling the number of images found), Crawl4AI is still faster than Firecrawl's simple crawl.\n",
+        "\n",
+        "Please note that actual performance may vary depending on network conditions and the specific content being crawled."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A6_1RK1_yZQO"
+      },
+      "source": [
+        "## Conclusion\n",
+        "\n",
+        "In this notebook, we've explored the powerful features of Crawl4AI, including:\n",
+        "\n",
+        "1. Basic crawling\n",
+        "2. JavaScript execution and CSS selector usage\n",
+        "3. Proxy support\n",
+        "4. Structured data extraction with OpenAI\n",
+        "5. Advanced multi-page crawling with JavaScript execution\n",
+        "6. Fast structured output using JsonCssExtractionStrategy\n",
+        "7. Speed comparison with other services\n",
+        "\n",
+        "Crawl4AI offers a fast, flexible, and powerful solution for web crawling and data extraction tasks. Its asynchronous architecture and advanced features make it suitable for a wide range of applications, from simple web scraping to complex, multi-page data extraction scenarios.\n",
+        "\n",
+        "For more information and advanced usage, please visit the [Crawl4AI documentation](https://docs.crawl4ai.com/).\n",
+        "\n",
+        "Happy crawling!"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "venv",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.13"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

docs/md_v2/advanced/network-console-capture.md

@@ -1,205 +0,0 @@
-# Network Requests & Console Message Capturing
-
-Crawl4AI can capture all network requests and browser console messages during a crawl, which is invaluable for debugging, security analysis, or understanding page behavior.
-
-## Configuration
-
-To enable network and console capturing, use these configuration options:
-
-```python
-from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-
-# Enable both network request capture and console message capture
-config = CrawlerRunConfig(
-    capture_network_requests=True,  # Capture all network requests and responses
-    capture_console_messages=True   # Capture all browser console output
-)
-```
-
-## Example Usage
-
-```python
-import asyncio
-import json
-from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-
-async def main():
-    # Enable both network request capture and console message capture
-    config = CrawlerRunConfig(
-        capture_network_requests=True,
-        capture_console_messages=True
-    )
-    
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun(
-            url="https://example.com",
-            config=config
-        )
-        
-        if result.success:
-            # Analyze network requests
-            if result.network_requests:
-                print(f"Captured {len(result.network_requests)} network events")
-                
-                # Count request types
-                request_count = len([r for r in result.network_requests if r.get("event_type") == "request"])
-                response_count = len([r for r in result.network_requests if r.get("event_type") == "response"])
-                failed_count = len([r for r in result.network_requests if r.get("event_type") == "request_failed"])
-                
-                print(f"Requests: {request_count}, Responses: {response_count}, Failed: {failed_count}")
-                
-                # Find API calls
-                api_calls = [r for r in result.network_requests 
-                            if r.get("event_type") == "request" and "api" in r.get("url", "")]
-                if api_calls:
-                    print(f"Detected {len(api_calls)} API calls:")
-                    for call in api_calls[:3]:  # Show first 3
-                        print(f"  - {call.get('method')} {call.get('url')}")
-            
-            # Analyze console messages
-            if result.console_messages:
-                print(f"Captured {len(result.console_messages)} console messages")
-                
-                # Group by type
-                message_types = {}
-                for msg in result.console_messages:
-                    msg_type = msg.get("type", "unknown")
-                    message_types[msg_type] = message_types.get(msg_type, 0) + 1
-                
-                print("Message types:", message_types)
-                
-                # Show errors (often the most important)
-                errors = [msg for msg in result.console_messages if msg.get("type") == "error"]
-                if errors:
-                    print(f"Found {len(errors)} console errors:")
-                    for err in errors[:2]:  # Show first 2
-                        print(f"  - {err.get('text', '')[:100]}")
-            
-            # Export all captured data to a file for detailed analysis
-            with open("network_capture.json", "w") as f:
-                json.dump({
-                    "url": result.url,
-                    "network_requests": result.network_requests or [],
-                    "console_messages": result.console_messages or []
-                }, f, indent=2)
-            
-            print("Exported detailed capture data to network_capture.json")
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-## Captured Data Structure
-
-### Network Requests
-
-The `result.network_requests` contains a list of dictionaries, each representing a network event with these common fields:
-
-| Field | Description |
-|-------|-------------|
-| `event_type` | Type of event: `"request"`, `"response"`, or `"request_failed"` |
-| `url` | The URL of the request |
-| `timestamp` | Unix timestamp when the event was captured |
-
-#### Request Event Fields
-
-```json
-{
-  "event_type": "request",
-  "url": "https://example.com/api/data.json",
-  "method": "GET",
-  "headers": {"User-Agent": "...", "Accept": "..."},
-  "post_data": "key=value&otherkey=value",
-  "resource_type": "fetch",
-  "is_navigation_request": false,
-  "timestamp": 1633456789.123
-}
-```
-
-#### Response Event Fields
-
-```json
-{
-  "event_type": "response",
-  "url": "https://example.com/api/data.json",
-  "status": 200,
-  "status_text": "OK",
-  "headers": {"Content-Type": "application/json", "Cache-Control": "..."},
-  "from_service_worker": false,
-  "request_timing": {"requestTime": 1234.56, "receiveHeadersEnd": 1234.78},
-  "timestamp": 1633456789.456
-}
-```
-
-#### Failed Request Event Fields
-
-```json
-{
-  "event_type": "request_failed",
-  "url": "https://example.com/missing.png",
-  "method": "GET",
-  "resource_type": "image",
-  "failure_text": "net::ERR_ABORTED 404",
-  "timestamp": 1633456789.789
-}
-```
-
-### Console Messages
-
-The `result.console_messages` contains a list of dictionaries, each representing a console message with these common fields:
-
-| Field | Description |
-|-------|-------------|
-| `type` | Message type: `"log"`, `"error"`, `"warning"`, `"info"`, etc. |
-| `text` | The message text |
-| `timestamp` | Unix timestamp when the message was captured |
-
-#### Console Message Example
-
-```json
-{
-  "type": "error",
-  "text": "Uncaught TypeError: Cannot read property 'length' of undefined",
-  "location": "https://example.com/script.js:123:45",
-  "timestamp": 1633456790.123
-}
-```
-
-## Key Benefits
-
-- **Full Request Visibility**: Capture all network activity including:
-  - Requests (URLs, methods, headers, post data)
-  - Responses (status codes, headers, timing)
-  - Failed requests (with error messages)
-  
-- **Console Message Access**: View all JavaScript console output:
-  - Log messages
-  - Warnings
-  - Errors with stack traces
-  - Developer debugging information
-
-- **Debugging Power**: Identify issues such as:
-  - Failed API calls or resource loading
-  - JavaScript errors affecting page functionality
-  - CORS or other security issues
-  - Hidden API endpoints and data flows
-
-- **Security Analysis**: Detect:
-  - Unexpected third-party requests
-  - Data leakage in request payloads
-  - Suspicious script behavior
-
-- **Performance Insights**: Analyze:
-  - Request timing data
-  - Resource loading patterns
-  - Potential bottlenecks
-
-## Use Cases
-
-1. **API Discovery**: Identify hidden endpoints and data flows in single-page applications
-2. **Debugging**: Track down JavaScript errors affecting page functionality
-3. **Security Auditing**: Detect unwanted third-party requests or data leakage
-4. **Performance Analysis**: Identify slow-loading resources
-5. **Ad/Tracker Analysis**: Detect and catalog advertising or tracking calls
-
-This capability is especially valuable for complex sites with heavy JavaScript, single-page applications, or when you need to understand the exact communication happening between a browser and servers.

docs/md_v2/api/crawl-result.md

@@ -15,7 +15,6 @@ class CrawlResult(BaseModel):
     downloaded_files: Optional[List[str]] = None
     screenshot: Optional[str] = None
     pdf : Optional[bytes] = None
-    mhtml: Optional[str] = None
     markdown: Optional[Union[str, MarkdownGenerationResult]] = None
     extracted_content: Optional[str] = None
     metadata: Optional[dict] = None
@@ -237,16 +236,7 @@ if result.pdf:
         f.write(result.pdf)
 ```
 
-### 5.5 **`mhtml`** *(Optional[str])*  
-**What**: MHTML snapshot of the page if `capture_mhtml=True` in `CrawlerRunConfig`. MHTML (MIME HTML) format preserves the entire web page with all its resources (CSS, images, scripts, etc.) in a single file.  
-**Usage**:
-```python
-if result.mhtml:
-    with open("page.mhtml", "w", encoding="utf-8") as f:
-        f.write(result.mhtml)
-```
-
-### 5.6 **`metadata`** *(Optional[dict])*  
+### 5.5 **`metadata`** *(Optional[dict])*  
 **What**: Page-level metadata if discovered (title, description, OG data, etc.).  
 **Usage**:
 ```python
@@ -281,69 +271,7 @@ for result in results:
 
 ---
 
-## 7. Network Requests & Console Messages
-
-When you enable network and console message capturing in `CrawlerRunConfig` using `capture_network_requests=True` and `capture_console_messages=True`, the `CrawlResult` will include these fields:
-
-### 7.1 **`network_requests`** *(Optional[List[Dict[str, Any]]])*
-**What**: A list of dictionaries containing information about all network requests, responses, and failures captured during the crawl.
-**Structure**:
-- Each item has an `event_type` field that can be `"request"`, `"response"`, or `"request_failed"`.
-- Request events include `url`, `method`, `headers`, `post_data`, `resource_type`, and `is_navigation_request`.
-- Response events include `url`, `status`, `status_text`, `headers`, and `request_timing`.
-- Failed request events include `url`, `method`, `resource_type`, and `failure_text`.
-- All events include a `timestamp` field.
-
-**Usage**:
-```python
-if result.network_requests:
-    # Count different types of events
-    requests = [r for r in result.network_requests if r.get("event_type") == "request"]
-    responses = [r for r in result.network_requests if r.get("event_type") == "response"]
-    failures = [r for r in result.network_requests if r.get("event_type") == "request_failed"]
-    
-    print(f"Captured {len(requests)} requests, {len(responses)} responses, and {len(failures)} failures")
-    
-    # Analyze API calls
-    api_calls = [r for r in requests if "api" in r.get("url", "")]
-    
-    # Identify failed resources
-    for failure in failures:
-        print(f"Failed to load: {failure.get('url')} - {failure.get('failure_text')}")
-```
-
-### 7.2 **`console_messages`** *(Optional[List[Dict[str, Any]]])*
-**What**: A list of dictionaries containing all browser console messages captured during the crawl.
-**Structure**:
-- Each item has a `type` field indicating the message type (e.g., `"log"`, `"error"`, `"warning"`, etc.).
-- The `text` field contains the actual message text.
-- Some messages include `location` information (URL, line, column).
-- All messages include a `timestamp` field.
-
-**Usage**:
-```python
-if result.console_messages:
-    # Count messages by type
-    message_types = {}
-    for msg in result.console_messages:
-        msg_type = msg.get("type", "unknown")
-        message_types[msg_type] = message_types.get(msg_type, 0) + 1
-    
-    print(f"Message type counts: {message_types}")
-    
-    # Display errors (which are usually most important)
-    for msg in result.console_messages:
-        if msg.get("type") == "error":
-            print(f"Error: {msg.get('text')}")
-```
-
-These fields provide deep visibility into the page's network activity and browser console, which is invaluable for debugging, security analysis, and understanding complex web applications.
-
-For more details on network and console capturing, see the [Network & Console Capture documentation](../advanced/network-console-capture.md).
-
----
-
-## 8. Example: Accessing Everything
+## 7. Example: Accessing Everything
 
 ```python
 async def handle_result(result: CrawlResult):
@@ -376,36 +304,16 @@ async def handle_result(result: CrawlResult):
     if result.extracted_content:
         print("Structured data:", result.extracted_content)
     
-    # Screenshot/PDF/MHTML
+    # Screenshot/PDF
     if result.screenshot:
         print("Screenshot length:", len(result.screenshot))
     if result.pdf:
         print("PDF bytes length:", len(result.pdf))
-    if result.mhtml:
-        print("MHTML length:", len(result.mhtml))
-        
-    # Network and console capturing
-    if result.network_requests:
-        print(f"Network requests captured: {len(result.network_requests)}")
-        # Analyze request types
-        req_types = {}
-        for req in result.network_requests:
-            if "resource_type" in req:
-                req_types[req["resource_type"]] = req_types.get(req["resource_type"], 0) + 1
-        print(f"Resource types: {req_types}")
-        
-    if result.console_messages:
-        print(f"Console messages captured: {len(result.console_messages)}")
-        # Count by message type
-        msg_types = {}
-        for msg in result.console_messages:
-            msg_types[msg.get("type", "unknown")] = msg_types.get(msg.get("type", "unknown"), 0) + 1
-        print(f"Message types: {msg_types}")
 ```
 
 ---
 
-## 9. Key Points & Future
+## 8. Key Points & Future
 
 1. **Deprecated legacy properties of CrawlResult**  
    - `markdown_v2` - Deprecated in v0.5. Just use `markdown`. It holds the `MarkdownGenerationResult` now!

docs/md_v2/api/parameters.md

@@ -140,7 +140,6 @@ If your page is a single-page app with repeated JS updates, set `js_only=True` i
 | **`screenshot_wait_for`**                  | `float or None`     | Extra wait time before the screenshot.                                                                    |
 | **`screenshot_height_threshold`**          | `int` (~20000)      | If the page is taller than this, alternate screenshot strategies are used.                                |
 | **`pdf`**                                  | `bool` (False)      | If `True`, returns a PDF in `result.pdf`.                                                                 |
-| **`capture_mhtml`**                        | `bool` (False)      | If `True`, captures an MHTML snapshot of the page in `result.mhtml`. MHTML includes all page resources (CSS, images, etc.) in a single file. |
 | **`image_description_min_word_threshold`** | `int` (~50)         | Minimum words for an image’s alt text or description to be considered valid.                              |
 | **`image_score_threshold`**                | `int` (~3)          | Filter out low-scoring images. The crawler scores images by relevance (size, context, etc.).              |
 | **`exclude_external_images`**              | `bool` (False)      | Exclude images from other domains.                                                                        |

docs/md_v2/ask_ai/ask-ai.css

@@ -1,444 +0,0 @@
-/* ==== File: docs/ask_ai/ask_ai.css ==== */
-
-/* --- Basic Reset & Font --- */
-body {
-    /* Attempt to inherit variables from parent window (iframe context) */
-    /* Fallback values if variables are not inherited */
-    --fallback-bg: #070708;
-    --fallback-font: #e8e9ed;
-    --fallback-secondary: #a3abba;
-    --fallback-primary: #50ffff;
-    --fallback-primary-dimmed: #09b5a5;
-    --fallback-border: #1d1d20;
-    --fallback-code-bg: #1e1e1e;
-    --fallback-invert-font: #222225;
-    --font-stack: dm, Monaco, Courier New, monospace, serif;
-
-    font-family: var(--font-stack, "Courier New", monospace); /* Use theme font stack */
-    background-color: var(--background-color, var(--fallback-bg));
-    color: var(--font-color, var(--fallback-font));
-    margin: 0;
-    padding: 0;
-    font-size: 14px; /* Match global font size */
-    line-height: 1.5em; /* Match global line height */
-    height: 100vh; /* Ensure body takes full height */
-    overflow: hidden; /* Prevent body scrollbars, panels handle scroll */
-    display: flex; /* Use flex for the main container */
-}
-
-a {
-    color: var(--secondary-color, var(--fallback-secondary));
-    text-decoration: none;
-    transition: color 0.2s;
-}
-a:hover {
-    color: var(--primary-color, var(--fallback-primary));
-}
-
-/* --- Main Container Layout --- */
-.ai-assistant-container {
-    display: flex;
-    width: 100%;
-    height: 100%;
-    background-color: var(--background-color, var(--fallback-bg));
-}
-
-/* --- Sidebar Styling --- */
-.sidebar {
-    flex-shrink: 0; /* Prevent sidebars from shrinking */
-    height: 100%;
-    display: flex;
-    flex-direction: column;
-    /* background-color: var(--code-bg-color, var(--fallback-code-bg)); */
-    overflow-y: hidden; /* Header fixed, list scrolls */
-}
-
-.left-sidebar {
-    flex-basis: 240px; /* Width of history panel */
-    border-right: 1px solid var(--progress-bar-background, var(--fallback-border));
-}
-
-.right-sidebar {
-    flex-basis: 280px; /* Width of citations panel */
-    border-left: 1px solid var(--progress-bar-background, var(--fallback-border));
-}
-
-.sidebar header {
-    padding: 0.6em 1em;
-    border-bottom: 1px solid var(--progress-bar-background, var(--fallback-border));
-    flex-shrink: 0;
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
-}
-
-.sidebar header h3 {
-    margin: 0;
-    font-size: 1.1em;
-    color: var(--font-color, var(--fallback-font));
-}
-
-.sidebar ul {
-    list-style: none;
-    padding: 0;
-    margin: 0;
-    overflow-y: auto; /* Enable scrolling for the list */
-    flex-grow: 1; /* Allow list to take remaining space */
-    padding: 0.5em 0;
-}
-
-.sidebar ul li {
-    padding: 0.3em 1em;
-}
-.sidebar ul li.no-citations,
-.sidebar ul li.no-history {
-    color: var(--secondary-color, var(--fallback-secondary));
-    font-style: italic;
-    font-size: 0.9em;
-    padding-left: 1em;
-}
-
-.sidebar ul li a {
-    color: var(--secondary-color, var(--fallback-secondary));
-    text-decoration: none;
-    display: block;
-    padding: 0.2em 0.5em;
-    border-radius: 3px;
-    transition: background-color 0.2s, color 0.2s;
-}
-
-.sidebar ul li a:hover {
-    color: var(--primary-color, var(--fallback-primary));
-    background-color: rgba(80, 255, 255, 0.08); /* Use primary color with alpha */
-}
-/* Style for active history item */
-#history-list li.active a {
-    color: var(--primary-dimmed-color, var(--fallback-primary-dimmed));
-    font-weight: bold;
-    background-color: rgba(80, 255, 255, 0.12);
-}
-
-/* --- Chat Panel Styling --- */
-#chat-panel {
-    flex-grow: 1; /* Take remaining space */
-    display: flex;
-    flex-direction: column;
-    height: 100%;
-    overflow: hidden; /* Prevent overflow, internal elements handle scroll */
-}
-
-#chat-messages {
-    flex-grow: 1;
-    overflow-y: auto; /* Scrollable chat history */
-    padding: 1em 1.5em;
-    border-bottom: 1px solid var(--progress-bar-background, var(--fallback-border));
-}
-
-.message {
-    margin-bottom: 1em;
-    padding: 0.8em 1.2em;
-    border-radius: 8px;
-    max-width: 90%; /* Slightly wider */
-    line-height: 1.6;
-    /* Apply pre-wrap for better handling of spaces/newlines AND wrapping */
-    white-space: pre-wrap;
-    word-wrap: break-word; /* Ensure long words break */
-}
-
-.user-message {
-    background-color: var(--progress-bar-background, var(--fallback-border)); /* User message background */
-    color: var(--font-color, var(--fallback-font));
-    margin-left: auto; /* Align user messages to the right */
-    text-align: left;
-}
-
-.ai-message {
-    background-color: var(--code-bg-color, var(--fallback-code-bg)); /* AI message background */
-    color: var(--font-color, var(--fallback-font));
-    margin-right: auto; /* Align AI messages to the left */
-    border: 1px solid var(--progress-bar-background, var(--fallback-border));
-}
-.ai-message.welcome-message {
-    border: none;
-    background-color: transparent;
-    max-width: 100%;
-    text-align: center;
-    color: var(--secondary-color, var(--fallback-secondary));
-    white-space: normal;
-}
-
-/* Styles for code within messages */
-.ai-message code {
-    background-color: var(--invert-font-color, var(--fallback-invert-font)) !important; /* Use light bg for code */
-    /* color: var(--background-color, var(--fallback-bg)) !important; Dark text */
-    padding: 0.1em 0.4em;
-    border-radius: 4px;
-    font-size: 0.9em;
-}
-.ai-message pre {
-    background-color: var(--invert-font-color, var(--fallback-invert-font)) !important;
-    color: var(--background-color, var(--fallback-bg)) !important;
-    padding: 1em;
-    border-radius: 5px;
-    overflow-x: auto;
-    margin: 0.8em 0;
-    white-space: pre;
-}
-.ai-message pre code {
-    background-color: transparent !important;
-    padding: 0;
-    font-size: inherit;
-}
-
-/* Override white-space for specific elements generated by Markdown */
-.ai-message p,
-.ai-message ul,
-.ai-message ol,
-.ai-message blockquote {
-    white-space: normal; /* Allow standard wrapping for block elements */
-}
-
-/* --- Markdown Element Styling within Messages --- */
-.message p {
-    margin-top: 0;
-    margin-bottom: 0.5em;
-}
-.message p:last-child {
-    margin-bottom: 0;
-}
-.message ul,
-.message ol {
-    margin: 0.5em 0 0.5em 1.5em;
-    padding: 0;
-}
-.message li {
-    margin-bottom: 0.2em;
-}
-
-/* Code block styling (adjusts previous rules slightly) */
-.message code {
-    /* Inline code */
-    background-color: var(--invert-font-color, var(--fallback-invert-font)) !important;
-    color: var(--font-color);
-    padding: 0.1em 0.4em;
-    border-radius: 4px;
-    font-size: 0.9em;
-    /* Ensure inline code breaks nicely */
-    word-break: break-all;
-    white-space: normal; /* Allow inline code to wrap if needed */
-}
-.message pre {
-    /* Code block container */
-    background-color: var(--invert-font-color, var(--fallback-invert-font)) !important;
-    color: var(--background-color, var(--fallback-bg)) !important;
-    padding: 1em;
-    border-radius: 5px;
-    overflow-x: auto;
-    margin: 0.8em 0;
-    font-size: 0.9em; /* Slightly smaller code blocks */
-}
-.message pre code {
-    /* Code within code block */
-    background-color: transparent !important;
-    padding: 0;
-    font-size: inherit;
-    word-break: normal; /* Don't break words in code blocks */
-    white-space: pre; /* Preserve whitespace strictly in code blocks */
-}
-
-/* Thinking indicator */
-.message-thinking {
-    display: inline-block;
-    width: 5px;
-    height: 5px;
-    background-color: var(--primary-color, var(--fallback-primary));
-    border-radius: 50%;
-    margin-left: 8px;
-    vertical-align: middle;
-    animation: thinking 1s infinite ease-in-out;
-}
-@keyframes thinking {
-    0%,
-    100% {
-        opacity: 0.5;
-        transform: scale(0.8);
-    }
-    50% {
-        opacity: 1;
-        transform: scale(1.2);
-    }
-}
-
-/* --- Thinking Indicator (Blinking Cursor Style) --- */
-.thinking-indicator-cursor {
-    display: inline-block;
-    width: 10px; /* Width of the cursor */
-    height: 1.1em; /* Match line height */
-    background-color: var(--primary-color, var(--fallback-primary));
-    margin-left: 5px;
-    vertical-align: text-bottom; /* Align with text baseline */
-    animation: blink-cursor 1s step-end infinite;
-}
-
-@keyframes blink-cursor {
-    from,
-    to {
-        background-color: transparent;
-    }
-    50% {
-        background-color: var(--primary-color, var(--fallback-primary));
-    }
-}
-
-#chat-input-area {
-    flex-shrink: 0; /* Prevent input area from shrinking */
-    padding: 1em 1.5em;
-    display: flex;
-    align-items: flex-end; /* Align items to bottom */
-    gap: 10px;
-    background-color: var(--code-bg-color, var(--fallback-code-bg)); /* Match sidebars */
-}
-
-#chat-input-area textarea {
-    flex-grow: 1;
-    padding: 0.8em 1em;
-    border: 1px solid var(--progress-bar-background, var(--fallback-border));
-    background-color: var(--background-color, var(--fallback-bg));
-    color: var(--font-color, var(--fallback-font));
-    border-radius: 5px;
-    resize: none; /* Disable manual resize */
-    font-family: inherit;
-    font-size: 1em;
-    line-height: 1.4;
-    max-height: 150px; /* Limit excessive height */
-    overflow-y: auto;
-    /* rows: 2; */
-}
-
-#chat-input-area button {
-    /* Basic button styling - maybe inherit from main theme? */
-    padding: 0.6em 1.2em;
-    border: 1px solid var(--primary-dimmed-color, var(--fallback-primary-dimmed));
-    background-color: var(--primary-dimmed-color, var(--fallback-primary-dimmed));
-    color: var(--background-color, var(--fallback-bg));
-    border-radius: 5px;
-    cursor: pointer;
-    font-size: 0.9em;
-    transition: background-color 0.2s, border-color 0.2s;
-    height: min-content; /* Align with bottom of textarea */
-}
-
-#chat-input-area button:hover {
-    background-color: var(--primary-color, var(--fallback-primary));
-    border-color: var(--primary-color, var(--fallback-primary));
-}
-#chat-input-area button:disabled {
-    opacity: 0.6;
-    cursor: not-allowed;
-}
-
-.loading-indicator {
-    font-size: 0.9em;
-    color: var(--secondary-color, var(--fallback-secondary));
-    margin-right: 10px;
-    align-self: center;
-}
-
-/* --- Buttons --- */
-/* Inherit some button styles if possible */
-.btn.btn-sm {
-    color: var(--font-color, var(--fallback-font));
-    padding: 0.2em 0.5em;
-    font-size: 0.8em;
-    border: 1px solid var(--secondary-color, var(--fallback-secondary));
-    background: none;
-    border-radius: 3px;
-    cursor: pointer;
-}
-.btn.btn-sm:hover {
-    border-color: var(--font-color, var(--fallback-font));
-    background-color: var(--progress-bar-background, var(--fallback-border));
-}
-
-/* --- Basic Responsiveness --- */
-@media screen and (max-width: 900px) {
-    .left-sidebar {
-        flex-basis: 200px; /* Shrink history */
-    }
-    .right-sidebar {
-        flex-basis: 240px; /* Shrink citations */
-    }
-}
-
-@media screen and (max-width: 768px) {
-    /* Stack layout on mobile? Or hide sidebars? Hiding for now */
-    .sidebar {
-        display: none; /* Hide sidebars on small screens */
-    }
-    /* Could add toggle buttons later */
-}
-
-
-/* ==== File: docs/ask_ai/ask-ai.css (Updates V4 - Delete Button) ==== */
-
-
-.sidebar ul li {
-    /* Use flexbox to align link and delete button */
-    display: flex;
-    justify-content: space-between;
-    align-items: center;
-    padding: 0; /* Remove padding from li, add to link/button */
-    margin: 0.1em 0; /* Small vertical margin */
-}
-
-.sidebar ul li a {
-    /* Link takes most space */
-    flex-grow: 1;
-    padding: 0.3em 0.5em 0.3em 1em; /* Adjust padding */
-    /* Make ellipsis work for long titles */
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-    /* Keep existing link styles */
-    color: var(--secondary-color, var(--fallback-secondary));
-    text-decoration: none;
-    display: block;
-    border-radius: 3px;
-    transition: background-color 0.2s, color 0.2s;
-}
-.sidebar ul li a:hover {
-    color: var(--primary-color, var(--fallback-primary));
-    background-color: rgba(80, 255, 255, 0.08);
-}
-
-/* Style for active history item's link */
-#history-list li.active a {
-    color: var(--primary-dimmed-color, var(--fallback-primary-dimmed));
-    font-weight: bold;
-    background-color: rgba(80, 255, 255, 0.12);
-}
-
-/* --- Delete Chat Button --- */
-.delete-chat-btn {
-    flex-shrink: 0; /* Don't shrink */
-    background: none;
-    border: none;
-    color: var(--secondary-color, var(--fallback-secondary));
-    cursor: pointer;
-    padding: 0.4em 0.8em; /* Padding around icon */
-    font-size: 0.9em;
-    opacity: 0.5; /* Dimmed by default */
-    transition: opacity 0.2s, color 0.2s;
-    margin-left: 5px; /* Space between link and button */
-    border-radius: 3px;
-}
-
-.sidebar ul li:hover .delete-chat-btn,
-.delete-chat-btn:hover {
-    opacity: 1; /* Show fully on hover */
-    color: var(--error-color, #ff3c74); /* Use error color on hover */
-}
-.delete-chat-btn:focus {
-    outline: 1px dashed var(--error-color, #ff3c74); /* Accessibility */
-     opacity: 1;
-}

docs/md_v2/ask_ai/ask-ai.js

@@ -1,603 +0,0 @@
-// ==== File: docs/ask_ai/ask-ai.js (Marked, Streaming, History) ====
-
-document.addEventListener("DOMContentLoaded", () => {
-    console.log("AI Assistant JS V2 Loaded");
-
-    // --- DOM Element Selectors ---
-    const historyList = document.getElementById("history-list");
-    const newChatButton = document.getElementById("new-chat-button");
-    const chatMessages = document.getElementById("chat-messages");
-    const chatInput = document.getElementById("chat-input");
-    const sendButton = document.getElementById("send-button");
-    const citationsList = document.getElementById("citations-list");
-
-    // --- Constants ---
-    const CHAT_INDEX_KEY = "aiAssistantChatIndex_v1";
-    const CHAT_PREFIX = "aiAssistantChat_v1_";
-
-    // --- State ---
-    let currentChatId = null;
-    let conversationHistory = []; // Holds message objects { sender: 'user'/'ai', text: '...' }
-    let isThinking = false;
-    let streamInterval = null; // To control the streaming interval
-
-    // --- Event Listeners ---
-    sendButton.addEventListener("click", handleSendMessage);
-    chatInput.addEventListener("keydown", handleInputKeydown);
-    newChatButton.addEventListener("click", handleNewChat);
-    chatInput.addEventListener("input", autoGrowTextarea);
-
-    // --- Initialization ---
-    loadChatHistoryIndex(); // Load history list on startup
-    const initialQuery = checkForInitialQuery(window.parent.location); // Check for query param
-    if (!initialQuery) {
-        loadInitialChat(); // Load normally if no query
-    }
-
-    // --- Core Functions ---
-
-    function handleSendMessage() {
-        const userMessageText = chatInput.value.trim();
-        if (!userMessageText || isThinking) return;
-
-        setThinking(true); // Start thinking state
-
-        // Add user message to state and UI
-        const userMessage = { sender: "user", text: userMessageText };
-        conversationHistory.push(userMessage);
-        addMessageToChat(userMessage, false); // Add user message without parsing markdown
-
-        chatInput.value = "";
-        autoGrowTextarea(); // Reset textarea height
-
-        // Prepare for AI response (create empty div)
-        const aiMessageDiv = addMessageToChat({ sender: "ai", text: "" }, true); // Add empty div with thinking indicator
-
-        // TODO: Generate fingerprint/JWT here
-
-        // TODO: Send `conversationHistory` + JWT to backend API
-        // Replace placeholder below with actual API call
-        // The backend should ideally return a stream of text tokens
-
-        // --- Placeholder Streaming Simulation ---
-        const simulatedFullResponse = `Okay, Here’s a minimal Python script that creates an AsyncWebCrawler, fetches a webpage, and prints the first 300 characters of its Markdown output:
-
-\`\`\`python
-import asyncio
-from crawl4ai import AsyncWebCrawler
-
-async def main():
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun("https://example.com")
-        print(result.markdown[:300])  # Print first 300 chars
-
-if __name__ == "__main__":
-    asyncio.run(main())
-\`\`\`
-
-A code snippet: \`crawler.run()\`. Check the [quickstart](/core/quickstart).`;
-
-        // Simulate receiving the response stream
-        streamSimulatedResponse(aiMessageDiv, simulatedFullResponse);
-
-        // // Simulate receiving citations *after* stream starts (or with first chunk)
-        // setTimeout(() => {
-        //     addCitations([
-        //         { title: "Simulated Doc 1", url: "#sim1" },
-        //         { title: "Another Concept", url: "#sim2" },
-        //     ]);
-        // }, 500); // Citations appear shortly after thinking starts
-    }
-
-    function handleInputKeydown(event) {
-        if (event.key === "Enter" && !event.shiftKey) {
-            event.preventDefault();
-            handleSendMessage();
-        }
-    }
-
-    function addMessageToChat(message, addThinkingIndicator = false) {
-        const messageDiv = document.createElement("div");
-        messageDiv.classList.add("message", `${message.sender}-message`);
-
-        // Parse markdown and set HTML
-        messageDiv.innerHTML = message.text ? marked.parse(message.text) : "";
-
-        if (message.sender === "ai") {
-            // Apply Syntax Highlighting AFTER setting innerHTML
-            messageDiv.querySelectorAll("pre code:not(.hljs)").forEach((block) => {
-                if (typeof hljs !== "undefined") {
-                    // Check if already highlighted to prevent double-highlighting issues
-                    if (!block.classList.contains("hljs")) {
-                        hljs.highlightElement(block);
-                    }
-                } else {
-                    console.warn("highlight.js (hljs) not found for syntax highlighting.");
-                }
-            });
-
-            // Add thinking indicator if needed (and not already present)
-            if (addThinkingIndicator && !message.text && !messageDiv.querySelector(".thinking-indicator-cursor")) {
-                const thinkingDiv = document.createElement("div");
-                thinkingDiv.className = "thinking-indicator-cursor";
-                messageDiv.appendChild(thinkingDiv);
-            }
-        } else {
-            // User messages remain plain text
-            // messageDiv.textContent = message.text;
-        }
-
-        // wrap each pre in a div.terminal
-        messageDiv.querySelectorAll("pre").forEach((block) => {
-            const wrapper = document.createElement("div");
-            wrapper.className = "terminal";
-            block.parentNode.insertBefore(wrapper, block);
-            wrapper.appendChild(block);
-        });
-
-        chatMessages.appendChild(messageDiv);
-        // Scroll only if user is near the bottom? (More advanced)
-        // Simple scroll for now:
-        scrollToBottom();
-        return messageDiv; // Return the created element
-    }
-
-    function streamSimulatedResponse(messageDiv, fullText) {
-        const thinkingIndicator = messageDiv.querySelector(".thinking-indicator-cursor");
-        if (thinkingIndicator) thinkingIndicator.remove();
-
-        const tokens = fullText.split(/(\s+)/);
-        let currentText = "";
-        let tokenIndex = 0;
-        // Clear previous interval just in case
-        if (streamInterval) clearInterval(streamInterval);
-
-        streamInterval = setInterval(() => {
-            const cursorSpan = '<span class="thinking-indicator-cursor"></span>'; // Cursor for streaming
-            if (tokenIndex < tokens.length) {
-                currentText += tokens[tokenIndex];
-                // Render intermediate markdown + cursor
-                messageDiv.innerHTML = marked.parse(currentText + cursorSpan);
-                // Re-highlight code blocks on each stream update - might be slightly inefficient
-                // but ensures partial code blocks look okay. Highlight only final on completion.
-                // messageDiv.querySelectorAll('pre code:not(.hljs)').forEach((block) => {
-                //     hljs.highlightElement(block);
-                // });
-                scrollToBottom(); // Keep scrolling as content streams
-                tokenIndex++;
-            } else {
-                // Streaming finished
-                clearInterval(streamInterval);
-                streamInterval = null;
-
-                // Final render without cursor
-                messageDiv.innerHTML = marked.parse(currentText);
-
-                // === Final Syntax Highlighting ===
-                messageDiv.querySelectorAll("pre code:not(.hljs)").forEach((block) => {
-                    if (typeof hljs !== "undefined" && !block.classList.contains("hljs")) {
-                        hljs.highlightElement(block);
-                    }
-                });
-
-                // === Extract Citations ===
-                const citations = extractMarkdownLinks(currentText);
-
-                // Wrap each pre in a div.terminal
-                messageDiv.querySelectorAll("pre").forEach((block) => {
-                    const wrapper = document.createElement("div");
-                    wrapper.className = "terminal";
-                    block.parentNode.insertBefore(wrapper, block);
-                    wrapper.appendChild(block);
-                });
-
-                const aiMessage = { sender: "ai", text: currentText, citations: citations };
-                conversationHistory.push(aiMessage);
-                updateCitationsDisplay();
-                saveCurrentChat();
-                setThinking(false);
-            }
-        }, 50); // Adjust speed
-    }
-
-    // === NEW Function to Extract Links ===
-    function extractMarkdownLinks(markdownText) {
-        const regex = /\[([^\]]+)\]\(([^)]+)\)/g; // [text](url)
-        const citations = [];
-        let match;
-        while ((match = regex.exec(markdownText)) !== null) {
-            // Avoid adding self-links from within the citations list if AI includes them
-            if (!match[2].startsWith("#citation-")) {
-                citations.push({
-                    title: match[1].trim(),
-                    url: match[2].trim(),
-                });
-            }
-        }
-        // Optional: Deduplicate links based on URL
-        const uniqueCitations = citations.filter(
-            (citation, index, self) => index === self.findIndex((c) => c.url === citation.url)
-        );
-        return uniqueCitations;
-    }
-
-    // === REVISED Function to Display Citations ===
-    function updateCitationsDisplay() {
-        let lastCitations = null;
-        // Find the most recent AI message with citations
-        for (let i = conversationHistory.length - 1; i >= 0; i--) {
-            if (
-                conversationHistory[i].sender === "ai" &&
-                conversationHistory[i].citations &&
-                conversationHistory[i].citations.length > 0
-            ) {
-                lastCitations = conversationHistory[i].citations;
-                break; // Found the latest citations
-            }
-        }
-
-        citationsList.innerHTML = ""; // Clear previous
-        if (!lastCitations) {
-            citationsList.innerHTML = '<li class="no-citations">No citations available.</li>';
-            return;
-        }
-
-        lastCitations.forEach((citation, index) => {
-            const li = document.createElement("li");
-            const a = document.createElement("a");
-            // Generate a unique ID for potential internal linking if needed
-            // a.id = `citation-${index}`;
-            a.href = citation.url || "#";
-            a.textContent = citation.title;
-            a.target = "_top"; // Open in main window
-            li.appendChild(a);
-            citationsList.appendChild(li);
-        });
-    }
-
-    function addCitations(citations) {
-        citationsList.innerHTML = ""; // Clear
-        if (!citations || citations.length === 0) {
-            citationsList.innerHTML = '<li class="no-citations">No citations available.</li>';
-            return;
-        }
-        citations.forEach((citation) => {
-            const li = document.createElement("li");
-            const a = document.createElement("a");
-            a.href = citation.url || "#";
-            a.textContent = citation.title;
-            a.target = "_top"; // Open in main window
-            li.appendChild(a);
-            citationsList.appendChild(li);
-        });
-    }
-
-    function setThinking(thinking) {
-        isThinking = thinking;
-        sendButton.disabled = thinking;
-        chatInput.disabled = thinking;
-        chatInput.placeholder = thinking ? "AI is responding..." : "Ask about Crawl4AI...";
-        // Stop any existing stream if we start thinking again (e.g., rapid resend)
-        if (thinking && streamInterval) {
-            clearInterval(streamInterval);
-            streamInterval = null;
-        }
-    }
-
-    function autoGrowTextarea() {
-        chatInput.style.height = "auto";
-        chatInput.style.height = `${chatInput.scrollHeight}px`;
-    }
-
-    function scrollToBottom() {
-        chatMessages.scrollTop = chatMessages.scrollHeight;
-    }
-
-    // --- Query Parameter Handling ---
-    function checkForInitialQuery(locationToCheck) {
-        // <-- Receive location object
-        if (!locationToCheck) {
-            console.warn("Ask AI: Could not access parent window location.");
-            return false;
-        }
-        const urlParams = new URLSearchParams(locationToCheck.search); // <-- Use passed location's search string
-        const encodedQuery = urlParams.get("qq"); // <-- Use 'qq'
-
-        if (encodedQuery) {
-            console.log("Initial query found (qq):", encodedQuery);
-            try {
-                const decodedText = decodeURIComponent(escape(atob(encodedQuery)));
-                console.log("Decoded query:", decodedText);
-
-                // Start new chat immediately
-                handleNewChat(true);
-
-                // Delay setting input and sending message slightly
-                setTimeout(() => {
-                    chatInput.value = decodedText;
-                    autoGrowTextarea();
-                    handleSendMessage();
-
-                    // Clean the PARENT window's URL
-                    try {
-                        const cleanUrl = locationToCheck.pathname;
-                        // Use parent's history object
-                        window.parent.history.replaceState({}, window.parent.document.title, cleanUrl);
-                    } catch (e) {
-                        console.warn("Ask AI: Could not clean parent URL using replaceState.", e);
-                        // This might fail due to cross-origin restrictions if served differently,
-                        // but should work fine with mkdocs serve on the same origin.
-                    }
-                }, 100);
-
-                return true; // Query processed
-            } catch (e) {
-                console.error("Error decoding initial query (qq):", e);
-                // Clean the PARENT window's URL even on error
-                try {
-                    const cleanUrl = locationToCheck.pathname;
-                    window.parent.history.replaceState({}, window.parent.document.title, cleanUrl);
-                } catch (cleanError) {
-                    console.warn("Ask AI: Could not clean parent URL after decode error.", cleanError);
-                }
-                return false;
-            }
-        }
-        return false; // No 'qq' query found
-    }
-
-    // --- History Management ---
-
-    function handleNewChat(isFromQuery = false) {
-        if (isThinking) return; // Don't allow new chat while responding
-
-        // Only save if NOT triggered immediately by a query parameter load
-        if (!isFromQuery) {
-            saveCurrentChat();
-        }
-
-        currentChatId = `chat_${Date.now()}`;
-        conversationHistory = []; // Clear message history state
-        chatMessages.innerHTML = ""; // Start with clean slate for query
-        if (!isFromQuery) {
-            // Show welcome only if manually started
-            chatMessages.innerHTML =
-                '<div class="message ai-message welcome-message">Started a new chat! Ask me anything about Crawl4AI.</div>';
-        }
-        addCitations([]); // Clear citations
-        updateCitationsDisplay(); // Clear UI
-
-        // Add to index and save
-        let index = loadChatIndex();
-        // Generate a generic title initially, update later
-        const newTitle = isFromQuery ? "Chat from Selection" : `Chat ${new Date().toLocaleString()}`;
-        // index.unshift({ id: currentChatId, title: `Chat ${new Date().toLocaleString()}` }); // Add to start
-        index.unshift({ id: currentChatId, title: newTitle });
-        saveChatIndex(index);
-
-        renderHistoryList(index); // Update UI
-        setActiveHistoryItem(currentChatId);
-        saveCurrentChat(); // Save the empty new chat state
-    }
-
-    function loadChat(chatId) {
-        if (isThinking || chatId === currentChatId) return;
-
-        // Check if chat data actually exists before proceeding
-        const storedChat = localStorage.getItem(CHAT_PREFIX + chatId);
-        if (storedChat === null) {
-            console.warn(`Attempted to load non-existent chat: ${chatId}. Removing from index.`);
-            deleteChatData(chatId); // Clean up index
-            loadChatHistoryIndex(); // Reload history list
-            loadInitialChat(); // Load next available chat
-            return;
-        }
-
-        console.log(`Loading chat: ${chatId}`);
-        saveCurrentChat(); // Save current before switching
-
-        try {
-            conversationHistory = JSON.parse(storedChat);
-            currentChatId = chatId;
-            renderChatMessages(conversationHistory);
-            updateCitationsDisplay();
-            setActiveHistoryItem(chatId);
-        } catch (e) {
-            console.error("Error loading chat:", chatId, e);
-            alert("Failed to load chat data.");
-            conversationHistory = [];
-            renderChatMessages(conversationHistory);
-            updateCitationsDisplay();
-        }
-    }
-
-    function saveCurrentChat() {
-        if (currentChatId && conversationHistory.length > 0) {
-            try {
-                localStorage.setItem(CHAT_PREFIX + currentChatId, JSON.stringify(conversationHistory));
-                console.log(`Chat ${currentChatId} saved.`);
-
-                // Update title in index (e.g., use first user message)
-                let index = loadChatIndex();
-                const currentItem = index.find((item) => item.id === currentChatId);
-                if (
-                    currentItem &&
-                    conversationHistory[0]?.sender === "user" &&
-                    !currentItem.title.startsWith("Chat about:")
-                ) {
-                    currentItem.title = `Chat about: ${conversationHistory[0].text.substring(0, 30)}...`;
-                    saveChatIndex(index);
-                    // Re-render history list if title changed - small optimization needed here maybe
-                    renderHistoryList(index);
-                    setActiveHistoryItem(currentChatId); // Re-set active after re-render
-                }
-            } catch (e) {
-                console.error("Error saving chat:", currentChatId, e);
-                // Handle potential storage full errors
-                if (e.name === "QuotaExceededError") {
-                    alert("Local storage is full. Cannot save chat history.");
-                    // Consider implementing history pruning logic here
-                }
-            }
-        } else if (currentChatId) {
-            // Save empty state for newly created chats if needed, or remove?
-            localStorage.setItem(CHAT_PREFIX + currentChatId, JSON.stringify([]));
-        }
-    }
-
-    function loadChatIndex() {
-        try {
-            const storedIndex = localStorage.getItem(CHAT_INDEX_KEY);
-            return storedIndex ? JSON.parse(storedIndex) : [];
-        } catch (e) {
-            console.error("Error loading chat index:", e);
-            return []; // Return empty array on error
-        }
-    }
-
-    function saveChatIndex(indexArray) {
-        try {
-            localStorage.setItem(CHAT_INDEX_KEY, JSON.stringify(indexArray));
-        } catch (e) {
-            console.error("Error saving chat index:", e);
-        }
-    }
-
-    function renderHistoryList(indexArray) {
-        historyList.innerHTML = ""; // Clear existing
-        if (!indexArray || indexArray.length === 0) {
-            historyList.innerHTML = '<li class="no-history">No past chats found.</li>';
-            return;
-        }
-        indexArray.forEach((item) => {
-            const li = document.createElement("li");
-            li.dataset.chatId = item.id; // Add ID to li for easier selection
-
-            const a = document.createElement("a");
-            a.href = "#";
-            a.dataset.chatId = item.id;
-            a.textContent = item.title || `Chat ${item.id.split("_")[1] || item.id}`;
-            a.title = a.textContent; // Tooltip for potentially long titles
-            a.addEventListener("click", (e) => {
-                e.preventDefault();
-                loadChat(item.id);
-            });
-
-            // === Add Delete Button ===
-            const deleteBtn = document.createElement("button");
-            deleteBtn.className = "delete-chat-btn";
-            deleteBtn.innerHTML = "✕"; // Trash can emoji/icon (or use text/SVG/FontAwesome)
-            deleteBtn.title = "Delete Chat";
-            deleteBtn.dataset.chatId = item.id; // Store ID on button too
-            deleteBtn.addEventListener("click", handleDeleteChat);
-
-            li.appendChild(a);
-            li.appendChild(deleteBtn); // Append button to the list item
-            historyList.appendChild(li);
-        });
-    }
-
-    function renderChatMessages(messages) {
-        chatMessages.innerHTML = ""; // Clear existing messages
-        messages.forEach((message) => {
-            // Ensure highlighting is applied when loading from history
-            addMessageToChat(message, false);
-        });
-        if (messages.length === 0) {
-            chatMessages.innerHTML =
-                '<div class="message ai-message welcome-message">Chat history loaded. Ask a question!</div>';
-        }
-        // Scroll to bottom after loading messages
-        scrollToBottom();
-    }
-
-    function setActiveHistoryItem(chatId) {
-        document.querySelectorAll("#history-list li").forEach((li) => li.classList.remove("active"));
-        // Select the LI element directly now
-        const activeLi = document.querySelector(`#history-list li[data-chat-id="${chatId}"]`);
-        if (activeLi) {
-            activeLi.classList.add("active");
-        }
-    }
-
-    function loadInitialChat() {
-        const index = loadChatIndex();
-        if (index.length > 0) {
-            loadChat(index[0].id);
-        } else {
-            // Check if handleNewChat wasn't already called by query handler
-            if (!currentChatId) {
-                handleNewChat();
-            }
-        }
-    }
-
-    function loadChatHistoryIndex() {
-        const index = loadChatIndex();
-        renderHistoryList(index);
-        if (currentChatId) setActiveHistoryItem(currentChatId);
-    }
-
-    // === NEW Function to Handle Delete Click ===
-    function handleDeleteChat(event) {
-        event.stopPropagation(); // Prevent triggering loadChat on the link behind it
-        const button = event.currentTarget;
-        const chatIdToDelete = button.dataset.chatId;
-
-        if (!chatIdToDelete) return;
-
-        // Confirmation dialog
-        if (
-            window.confirm(
-                `Are you sure you want to delete this chat session?\n"${
-                    button.previousElementSibling?.textContent || "Chat " + chatIdToDelete
-                }"`
-            )
-        ) {
-            console.log(`Deleting chat: ${chatIdToDelete}`);
-
-            // Perform deletion
-            const updatedIndex = deleteChatData(chatIdToDelete);
-
-            // If the deleted chat was the currently active one, load another chat
-            if (currentChatId === chatIdToDelete) {
-                currentChatId = null; // Reset current ID
-                conversationHistory = []; // Clear state
-                if (updatedIndex.length > 0) {
-                    // Load the new top chat (most recent remaining)
-                    loadChat(updatedIndex[0].id);
-                } else {
-                    // No chats left, start a new one
-                    handleNewChat();
-                }
-            } else {
-                // If a different chat was deleted, just re-render the list
-                renderHistoryList(updatedIndex);
-                // Re-apply active state in case IDs shifted (though they shouldn't)
-                setActiveHistoryItem(currentChatId);
-            }
-        }
-    }
-
-    // === NEW Function to Delete Chat Data ===
-    function deleteChatData(chatId) {
-        // Remove chat data
-        localStorage.removeItem(CHAT_PREFIX + chatId);
-
-        // Update index
-        let index = loadChatIndex();
-        index = index.filter((item) => item.id !== chatId);
-        saveChatIndex(index);
-
-        console.log(`Chat ${chatId} data and index entry removed.`);
-        return index; // Return the updated index
-    }
-
-    // --- Virtual Scrolling Placeholder ---
-    // NOTE: Virtual scrolling is complex. For now, we do direct rendering.
-    // If performance becomes an issue with very long chats/history,
-    // investigate libraries like 'simple-virtual-scroll' or 'virtual-scroller'.
-    // You would replace parts of `renderChatMessages` and `renderHistoryList`
-    // to work with the chosen library's API (providing data and item renderers).
-    console.warn("Virtual scrolling not implemented. Performance may degrade with very long chat histories.");
-});

docs/md_v2/ask_ai/index.html

@@ -1,64 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Crawl4AI Assistant</title>
-    <!-- Link main styles first for variable access -->
-    <link rel="stylesheet" href="../assets/layout.css">
-    <link rel="stylesheet" href="../assets/styles.css">
-    <!-- Link specific AI styles -->
-    <link rel="stylesheet" href="../assets/highlight.css">
-    <link rel="stylesheet" href="ask-ai.css">
-</head>
-<body>
-    <div class="ai-assistant-container">
-
-        <!-- Left Sidebar: Conversation History -->
-        <aside id="history-panel" class="sidebar left-sidebar">
-            <header>
-                <h3>History</h3>
-                <button id="new-chat-button" class="btn btn-sm">New Chat</button>
-            </header>
-            <ul id="history-list">
-                <!-- History items populated by JS -->
-            </ul>
-        </aside>
-
-        <!-- Main Area: Chat Interface -->
-        <main id="chat-panel">
-            <div id="chat-messages">
-                <!-- Chat messages populated by JS -->
-                 <div class="message ai-message welcome-message">
-                    Welcome to the Crawl4AI Assistant! How can I help you today?
-                 </div>
-            </div>
-            <div id="chat-input-area">
-                <!-- Loading indicator for general waiting (optional) -->
-                <!-- <div class="loading-indicator" style="display: none;">Thinking...</div> -->
-                <textarea id="chat-input" placeholder="Ask about Crawl4AI..." rows="2"></textarea> 
-                <button id="send-button">Send</button>
-            </div>
-        </main>
-
-        <!-- Right Sidebar: Citations / Context -->
-        <aside id="citations-panel" class="sidebar right-sidebar">
-            <header>
-                <h3>Citations</h3>
-            </header>
-            <ul id="citations-list">
-                <!-- Citations populated by JS -->
-                <li class="no-citations">No citations for this response yet.</li>
-            </ul>
-        </aside>
-
-    </div>
-
-    <!-- Include Marked.js library -->
-    <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
-    <script src="../assets/highlight.min.js"></script> 
-
-    <!-- Your AI Assistant Logic -->
-    <script src="ask-ai.js"></script>
-</body>
-</html>

docs/md_v2/assets/copy_code.js

@@ -1,62 +0,0 @@
-// ==== File: docs/assets/copy_code.js ====
-
-document.addEventListener('DOMContentLoaded', () => {
-    // Target specifically code blocks within the main content area
-    const codeBlocks = document.querySelectorAll('#terminal-mkdocs-main-content pre > code');
-
-    codeBlocks.forEach((codeElement) => {
-        const preElement = codeElement.parentElement; // The <pre> tag
-
-        // Ensure the <pre> tag can contain a positioned button
-        if (window.getComputedStyle(preElement).position === 'static') {
-            preElement.style.position = 'relative';
-        }
-
-        // Create the button
-        const copyButton = document.createElement('button');
-        copyButton.className = 'copy-code-button';
-        copyButton.type = 'button';
-        copyButton.setAttribute('aria-label', 'Copy code to clipboard');
-        copyButton.title = 'Copy code to clipboard';
-        copyButton.innerHTML = 'Copy'; // Or use an icon like an SVG or FontAwesome class
-
-        // Append the button to the <pre> element
-        preElement.appendChild(copyButton);
-
-        // Add click event listener
-        copyButton.addEventListener('click', () => {
-            copyCodeToClipboard(codeElement, copyButton);
-        });
-    });
-
-    async function copyCodeToClipboard(codeElement, button) {
-        // Use innerText to get the rendered text content, preserving line breaks
-        const textToCopy = codeElement.innerText;
-
-        try {
-            await navigator.clipboard.writeText(textToCopy);
-
-            // Visual feedback
-            button.innerHTML = 'Copied!';
-            button.classList.add('copied');
-            button.disabled = true; // Temporarily disable
-
-            // Revert button state after a short delay
-            setTimeout(() => {
-                button.innerHTML = 'Copy';
-                button.classList.remove('copied');
-                button.disabled = false;
-            }, 2000); // Show "Copied!" for 2 seconds
-
-        } catch (err) {
-            console.error('Failed to copy code: ', err);
-            // Optional: Provide error feedback on the button
-            button.innerHTML = 'Error';
-            setTimeout(() => {
-                button.innerHTML = 'Copy';
-            }, 2000);
-        }
-    }
-
-    console.log("Copy Code Button script loaded.");
-});

docs/md_v2/assets/floating_ask_ai_button.js

@@ -1,39 +0,0 @@
-// ==== File: docs/assets/floating_ask_ai_button.js ====
-
-document.addEventListener('DOMContentLoaded', () => {
-    const askAiPagePath = '/core/ask-ai/'; // IMPORTANT: Adjust this path if needed!
-    const currentPath = window.location.pathname;
-
-    // Determine the base URL for constructing the link correctly,
-    // especially if deployed in a sub-directory.
-    // This assumes a simple structure; adjust if needed.
-    const baseUrl = window.location.origin + (currentPath.startsWith('/core/') ? '../..' : '');
-
-
-    // Check if the current page IS the Ask AI page
-    // Use includes() for flexibility (handles trailing slash or .html)
-    if (currentPath.includes(askAiPagePath.replace(/\/$/, ''))) { // Remove trailing slash for includes check
-        console.log("Floating Ask AI Button: Not adding button on the Ask AI page itself.");
-        return; // Don't add the button on the target page
-    }
-
-    // --- Create the button ---
-    const fabLink = document.createElement('a');
-    fabLink.className = 'floating-ask-ai-button';
-    fabLink.href = askAiPagePath; // Construct the correct URL
-    fabLink.title = 'Ask Crawl4AI Assistant';
-    fabLink.setAttribute('aria-label', 'Ask Crawl4AI Assistant');
-
-    // Add content (using SVG icon for better visuals)
-    fabLink.innerHTML = `
-        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24" fill="currentColor">
-            <path d="M20 2H4c-1.1 0-2 .9-2 2v12c0 1.1.9 2 2 2h14l4 4V4c0-1.1-.9-2-2-2zm-2 12H6v-2h12v2zm0-3H6V9h12v2zm0-3H6V6h12v2z"/>
-        </svg>
-        <span>Ask AI</span>
-    `;
-
-    // Append to body
-    document.body.appendChild(fabLink);
-
-    console.log("Floating Ask AI Button added.");
-});

docs/md_v2/assets/github_stats.js

@@ -1,119 +0,0 @@
-// ==== File: assets/github_stats.js ====
-
-document.addEventListener('DOMContentLoaded', async () => {
-    // --- Configuration ---
-    const targetHeaderSelector = '.terminal .container:first-child'; // Selector for your header container
-    const insertBeforeSelector = '.terminal-nav'; // Selector for the element to insert the badge BEFORE (e.g., the main nav)
-                                                  // Or set to null to append at the end of the header.
-
-    // --- Find elements ---
-    const headerContainer = document.querySelector(targetHeaderSelector);
-    if (!headerContainer) {
-        console.warn('GitHub Stats: Header container not found with selector:', targetHeaderSelector);
-        return;
-    }
-
-    const repoLinkElement = headerContainer.querySelector('a[href*="github.com/"]'); // Find the existing GitHub link
-    let repoUrl = 'https://github.com/unclecode/crawl4ai';
-    // if (repoLinkElement) {
-    //     repoUrl = repoLinkElement.href;
-    // } else {
-    //     // Fallback: Try finding from config (requires template injection - harder)
-    //     // Or hardcode if necessary, but reading from the link is better.
-    //      console.warn('GitHub Stats: GitHub repo link not found in header.');
-    //      // Try to get repo_url from mkdocs config if available globally (less likely)
-    //      // repoUrl = window.mkdocs_config?.repo_url; // Requires setting this variable
-    //      // if (!repoUrl) return; // Exit if still no URL
-    //      return; // Exit for now if link isn't found
-    // }
-
-
-    // --- Extract Repo Owner/Name ---
-    let owner = '';
-    let repo = '';
-    try {
-        const url = new URL(repoUrl);
-        const pathParts = url.pathname.split('/').filter(part => part.length > 0);
-        if (pathParts.length >= 2) {
-            owner = pathParts[0];
-            repo = pathParts[1];
-        }
-    } catch (e) {
-        console.error('GitHub Stats: Could not parse repository URL:', repoUrl, e);
-        return;
-    }
-
-    if (!owner || !repo) {
-        console.warn('GitHub Stats: Could not extract owner/repo from URL:', repoUrl);
-        return;
-    }
-
-    // --- Get Version (Attempt to extract from site title) ---
-    let version = '';
-    const siteTitleElement = headerContainer.querySelector('.terminal-title, .site-title'); // Adjust selector based on theme's title element
-    // Example title: "Crawl4AI Documentation (v0.5.x)"
-    if (siteTitleElement) {
-         const match = siteTitleElement.textContent.match(/\((v?[^)]+)\)/); // Look for text in parentheses starting with 'v' (optional)
-         if (match && match[1]) {
-             version = match[1].trim();
-         }
-    }
-     if (!version) {
-        console.info('GitHub Stats: Could not extract version from title. You might need to adjust the selector or regex.');
-        // You could fallback to config.extra.version if injected into JS
-        // version = window.mkdocs_config?.extra?.version || 'N/A';
-     }
-
-
-    // --- Fetch GitHub API Data ---
-    let stars = '...';
-    let forks = '...';
-    try {
-        const apiUrl = `https://api.github.com/repos/${owner}/${repo}`;
-        const response = await fetch(apiUrl);
-
-        if (response.ok) {
-            const data = await response.json();
-            // Format large numbers (optional)
-            stars = data.stargazers_count > 1000 ? `${(data.stargazers_count / 1000).toFixed(1)}k` : data.stargazers_count;
-            forks = data.forks_count > 1000 ? `${(data.forks_count / 1000).toFixed(1)}k` : data.forks_count;
-        } else {
-            console.warn(`GitHub Stats: API request failed with status ${response.status}. Rate limit exceeded?`);
-            stars = 'N/A';
-            forks = 'N/A';
-        }
-    } catch (error) {
-        console.error('GitHub Stats: Error fetching repository data:', error);
-        stars = 'N/A';
-        forks = 'N/A';
-    }
-
-    // --- Create Badge HTML ---
-    const badgeContainer = document.createElement('div');
-    badgeContainer.className = 'github-stats-badge';
-
-    // Use innerHTML for simplicity, including potential icons (requires FontAwesome or similar)
-    // Ensure your theme loads FontAwesome or add it yourself if you want icons.
-    badgeContainer.innerHTML = `
-        <a href="${repoUrl}" target="_blank" rel="noopener">
-            <!-- Optional Icon (FontAwesome example) -->
-            <!-- <i class="fab fa-github"></i> -->
-             <span class="repo-name">${owner}/${repo}</span>
-             ${version ? `<span class="stat version"><i class="fas fa-tag"></i> ${version}</span>` : ''}
-            <span class="stat stars"><i class="fas fa-star"></i> ${stars}</span>
-            <span class="stat forks"><i class="fas fa-code-branch"></i> ${forks}</span>
-        </a>
-    `;
-
-    // --- Inject Badge into Header ---
-    const insertBeforeElement = insertBeforeSelector ? headerContainer.querySelector(insertBeforeSelector) : null;
-    if (insertBeforeElement) {
-        // headerContainer.insertBefore(badgeContainer, insertBeforeElement);
-        headerContainer.querySelector(insertBeforeSelector).appendChild(badgeContainer); 
-    } else {
-        headerContainer.appendChild(badgeContainer); 
-    }
-
-     console.info('GitHub Stats: Badge added to header.');
-
-});

docs/md_v2/assets/layout.css

@@ -1,441 +0,0 @@
-/* ==== File: assets/layout.css (Non-Fluid Centered Layout) ==== */
-
-:root {
-    --header-height: 55px; /* Adjust if needed */
-    --sidebar-width: 280px; /* Adjust if needed */
-    --toc-width: 340px; /* As specified */
-    --content-max-width: 90em; /* Max width for the centered content */
-    --layout-transition-speed: 0.2s;
-    --global-space: 10px;
-}
-
-/* --- Basic Setup --- */
-html {
-    scroll-behavior: smooth;
-    scroll-padding-top: calc(var(--header-height) + 15px);
-    box-sizing: border-box;
-}
-*, *:before, *:after {
-    box-sizing: inherit;
-}
-
-body {
-    padding-top: 0;
-    padding-bottom: 0;
-    background-color: var(--background-color);
-    color: var(--font-color);
-    /* Prevents horizontal scrollbars during transitions */
-    overflow-x: hidden;
-}
-
-/* --- Fixed Header --- */
-/* Full width, fixed header */
-.terminal .container:first-child { /* Assuming this targets the header container */
-    position: fixed;
-    top: 0;
-    left: 0;
-    right: 0;
-    height: var(--header-height);
-    background-color: var(--background-color);
-    z-index: 1000;
-    border-bottom: 1px solid var(--progress-bar-background);
-    max-width: none; /* Override any container max-width */
-    padding: 0 calc(var(--global-space) * 2);
-}
-
-/* --- Main Layout Container (Below Header) --- */
-/* This container just provides space for the fixed header */
-.container:has(.terminal-mkdocs-main-grid) {
-    margin: 0 auto;
-    padding: 0;
-    padding-top: var(--header-height); /* Space for fixed header */
-}
-
-/* --- Flex Container: Grid holding content and toc (CENTERED) --- */
-/* THIS is the main centered block */
-.terminal-mkdocs-main-grid {
-    display: flex;
-    align-items: flex-start;
-    /* Enforce max-width and center */
-    max-width: var(--content-max-width);
-    margin-left: auto;
-    margin-right: auto;
-    position: relative;
-    /* Apply side padding within the centered block */
-    padding-left: calc(var(--global-space) * 2);
-    padding-right: calc(var(--global-space) * 2);
-    /* Add margin-left to clear the fixed sidebar */
-    margin-left: var(--sidebar-width);
-}
-
-/* --- 1. Fixed Left Sidebar (Viewport Relative) --- */
-#terminal-mkdocs-side-panel {
-    position: fixed;
-    top: var(--header-height);
-    left: max(0px, calc((90vw - var(--content-max-width)) / 2)); 
-    bottom: 0;
-    width: var(--sidebar-width);
-    background-color: var(--background-color);
-    border-right: 1px solid var(--progress-bar-background);
-    overflow-y: auto;
-    z-index: 900;
-    padding: 1em calc(var(--global-space) * 2);
-    padding-bottom: 2em;
-    /* transition: left var(--layout-transition-speed) ease-in-out; */
-}
-
-/* --- 2. Main Content Area (Within Centered Grid) --- */
-#terminal-mkdocs-main-content {
-    flex-grow: 1;
-    flex-shrink: 1;
-    min-width: 0; /* Flexbox shrink fix */
-
-    /* No left/right margins needed here - handled by parent grid */
-    margin-left: 0;
-    margin-right: 0;
-
-    /* Internal Padding */
-    padding: 1.5em 2em;
-
-    position: relative;
-    z-index: 1;
-}
-
-/* --- 3. Right Table of Contents (Sticky, Within Centered Grid) --- */
-#toc-sidebar {
-    flex-basis: var(--toc-width);
-    flex-shrink: 0;
-    width: var(--toc-width);
-
-    position: sticky; /* Sticks within the centered grid */
-    top: var(--header-height);
-    align-self: stretch;
-    height: calc(100vh - var(--header-height));
-    overflow-y: auto;
-
-    padding: 1.5em 1em;
-    font-size: 0.85em;
-    border-left: 1px solid var(--progress-bar-background);
-    z-index: 800;
-    /* display: none; /* JS handles */
-}
-
-/* (ToC link styles remain the same) */
-#toc-sidebar h4 { margin-top: 0; margin-bottom: 1em; font-size: 1.1em; color: var(--secondary-color); padding-left: 0.8em; }
-#toc-sidebar ul { list-style: none; padding: 0; margin: 0; }
-#toc-sidebar ul li a { display: block; padding: 0.3em 0; color: var(--secondary-color); text-decoration: none; border-left: 3px solid transparent; padding-left: 0.8em; transition: all 0.1s ease-in-out; line-height: 1.4; word-break: break-word; }
-#toc-sidebar ul li.toc-level-3 a { padding-left: 1.8em; }
-#toc-sidebar ul li.toc-level-4 a { padding-left: 2.8em; }
-#toc-sidebar ul li a:hover { color: var(--font-color); background-color: rgba(255, 255, 255, 0.05); }
-#toc-sidebar ul li a.active { color: var(--primary-color); border-left-color: var(--primary-color); background-color: rgba(80, 255, 255, 0.08); }
-
-
-/* --- Footer Styling (Respects Centered Layout) --- */
-footer {
-    background-color: var(--code-bg-color);
-    color: var(--secondary-color);
-    position: relative;
-    z-index: 10;
-    margin-top: 2em;
-
-    /* Apply margin-left to clear the fixed sidebar */
-    margin-left: var(--sidebar-width);
-
-    /* Constrain width relative to the centered grid it follows */
-    max-width: calc(var(--content-max-width) - var(--sidebar-width));
-    margin-right: auto; /* Keep it left-aligned within the space next to sidebar */
-
-    /* Use padding consistent with the grid */
-    padding: 2em calc(var(--global-space) * 2);
-}
-
-/* Adjust footer grid if needed */
-.terminal-mkdocs-footer-grid {
-    display: grid;
-    grid-template-columns: 1fr auto;
-    gap: 1em;
-    align-items: center;
-}
-
-/* ==========================================================================
-   RESPONSIVENESS (Adapting the Non-Fluid Layout)
-   ========================================================================== */
-
-/* --- Medium screens: Hide ToC --- */
-@media screen and (max-width: 1200px) {
-    #toc-sidebar {
-        display: none;
-    }
-
-    .terminal-mkdocs-main-grid {
-        /* Grid adjusts automatically as ToC is removed */
-        /* Ensure grid padding remains */
-         padding-left: calc(var(--global-space) * 2);
-         padding-right: calc(var(--global-space) * 2);
-    }
-
-    #terminal-mkdocs-main-content {
-        /* Content area naturally expands */
-    }
-
-    footer {
-        /* Footer still respects the left sidebar and overall max width */
-        margin-left: var(--sidebar-width);
-        max-width: calc(var(--content-max-width) - var(--sidebar-width));
-        /* Padding remains consistent */
-         padding-left: calc(var(--global-space) * 2);
-         padding-right: calc(var(--global-space) * 2);
-    }
-}
-
-/* --- Small screens: Hide left sidebar, full width content & footer --- */
-@media screen and (max-width: 768px) {
-
-    #terminal-mkdocs-side-panel {
-        left: calc(-1 * var(--sidebar-width));
-        z-index: 1100;
-        box-shadow: 2px 0 10px rgba(0,0,0,0.3);
-    }
-    #terminal-mkdocs-side-panel.sidebar-visible {
-        left: 0;
-    }
-
-    .terminal-mkdocs-main-grid {
-        /* Grid now takes full width (minus body padding) */
-        margin-left: 0; /* Override sidebar margin */
-        margin-right: 0; /* Override auto margin */
-        max-width: 100%; /* Allow full width */
-        padding-left: var(--global-space); /* Reduce padding */
-        padding-right: var(--global-space);
-    }
-
-    #terminal-mkdocs-main-content {
-        padding: 1.5em 1em; /* Adjust internal padding */
-    }
-
-    footer {
-        margin-left: 0; /* Full width footer */
-        max-width: 100%; /* Allow full width */
-        padding: 2em 1em; /* Adjust internal padding */
-    }
-
-    .terminal-mkdocs-footer-grid {
-         grid-template-columns: 1fr; /* Stack footer items */
-         text-align: center;
-         gap: 0.5em;
-    }
-    /* Remember JS for toggle button & overlay */
-}
-
-
-/* ==== GitHub Stats Badge Styling ==== */
-
-.github-stats-badge {
-    display: inline-block; /* Or flex if needed */
-    margin-left: 2em; /* Adjust spacing */
-    vertical-align: middle; /* Align with other header items */
-    font-size: 0.9em; /* Slightly smaller font */
-}
-
-.github-stats-badge a {
-    color: var(--secondary-color); /* Use secondary color */
-    text-decoration: none;
-    display: flex; /* Use flex for alignment */
-    align-items: center;
-    gap: 0.8em; /* Space between items */
-    padding: 0.2em 0.5em;
-    border: 1px solid var(--progress-bar-background); /* Subtle border */
-    border-radius: 4px;
-    transition: color 0.2s, background-color 0.2s;
-}
-
-.github-stats-badge a:hover {
-    color: var(--font-color); /* Brighter color on hover */
-    background-color: var(--progress-bar-background); /* Subtle background on hover */
-}
-
-.github-stats-badge .repo-name {
-    color: var(--font-color); /* Make repo name stand out slightly */
-    font-weight: 500; /* Optional bolder weight */
-}
-
-.github-stats-badge .stat {
-    /* Styles for individual stats (version, stars, forks) */
-    white-space: nowrap; /* Prevent wrapping */
-}
-
-.github-stats-badge .stat i {
-    /* Optional: Style for FontAwesome icons */
-    margin-right: 0.3em;
-    color: var(--secondary-dimmed-color); /* Dimmer color for icons */
-}
-
-
-/* Adjust positioning relative to search/nav if needed */
-/* Example: If search is floated right */
-/* .terminal-nav { float: left; } */
-/* .github-stats-badge { float: left; } */
-/* #mkdocs-search-query { float: right; } */
-
-/* --- Responsive adjustments --- */
-@media screen and (max-width: 900px) { /* Example breakpoint */
-    .github-stats-badge .repo-name {
-        display: none; /* Hide full repo name on smaller screens */
-    }
-    .github-stats-badge {
-        margin-left: 1em;
-    }
-     .github-stats-badge a {
-        gap: 0.5em;
-    }
-}
-@media screen and (max-width: 768px) {
-    /* Further hide or simplify on mobile if needed */
-     .github-stats-badge {
-        display: none; /* Example: Hide completely on smallest screens */
-     }
-}
-
-/* --- Ask AI Selection Button --- */
-.ask-ai-selection-button {
-    background-color: var(--primary-dimmed-color, #09b5a5);
-    color: var(--background-color, #070708);
-    border: none;
-    padding: 4px 8px;
-    font-size: 0.8em;
-    border-radius: 4px;
-    cursor: pointer;
-    box-shadow: 0 2px 5px rgba(0, 0, 0, 0.3);
-    transition: background-color 0.2s ease;
-    white-space: nowrap;
-}
-
-.ask-ai-selection-button:hover {
-    background-color: var(--primary-color, #50ffff);
-}
-
-/* ==== File: docs/assets/layout.css (Additions) ==== */
-
-/* ... (keep all existing layout CSS) ... */
-
-/* --- Copy Code Button Styling --- */
-
-/* Ensure the parent <pre> can contain the absolutely positioned button */
-#terminal-mkdocs-main-content pre {
-    position: relative; /* Needed for absolute positioning of child */
-    /* Add a little padding top/right to make space for the button */
-    padding-top: 2.5em;
-    padding-right: 1em; /* Ensure padding is sufficient */
-}
-
-.copy-code-button {
-    position: absolute;
-    top: 0.5em; /* Adjust spacing from top */
-    left: 0.5em; /* Adjust spacing from left */
-    z-index: 1; /* Sit on top of code */
-
-    background-color: var(--progress-bar-background, #444); /* Use a background */
-    color: var(--font-color, #eaeaea);
-    border: 1px solid var(--secondary-color, #727578);
-    padding: 3px 8px;
-    font-size: 0.8em;
-    font-family: var(--font-stack, monospace);
-    border-radius: 4px;
-    cursor: pointer;
-    opacity: 0; /* Hidden by default */
-    transition: opacity 0.2s ease-in-out, background-color 0.2s ease, color 0.2s ease;
-    white-space: nowrap;
-}
-
-/* Show button on hover of the <pre> container */
-#terminal-mkdocs-main-content pre:hover .copy-code-button {
-    opacity: 0.8; /* Show partially */
-}
-
-.copy-code-button:hover {
-    opacity: 1; /* Fully visible on button hover */
-    background-color: var(--secondary-color, #727578);
-}
-
-.copy-code-button:focus {
-     opacity: 1; /* Ensure visible when focused */
-     outline: 1px dashed var(--primary-color);
-}
-
-
-/* Style for "Copied!" state */
-.copy-code-button.copied {
-    background-color: var(--primary-dimmed-color, #09b5a5);
-    color: var(--background-color, #070708);
-    border-color: var(--primary-dimmed-color, #09b5a5);
-    opacity: 1; /* Ensure visible */
-}
-.copy-code-button.copied:hover {
-     background-color: var(--primary-dimmed-color, #09b5a5); /* Prevent hover change */
-}
-
-/* ==== File: docs/assets/layout.css (Additions) ==== */
-
-/* ... (keep all existing layout CSS) ... */
-
-/* --- Floating Ask AI Button --- */
-.floating-ask-ai-button {
-    position: fixed;
-    bottom: 25px;
-    right: 25px;
-    z-index: 1050; /* Below modals, above most content */
-
-    background-color: var(--primary-dimmed-color, #09b5a5);
-    color: var(--background-color, #070708);
-    border: none;
-    border-radius: 50%; /* Make it circular */
-    width: 60px; /* Adjust size */
-    height: 60px; /* Adjust size */
-    padding: 10px; /* Adjust padding */
-    box-shadow: 0 4px 10px rgba(0, 0, 0, 0.4);
-    cursor: pointer;
-    transition: background-color 0.2s ease, transform 0.2s ease;
-
-    display: flex;
-    flex-direction: column; /* Stack icon and text */
-    align-items: center;
-    justify-content: center;
-    text-decoration: none;
-    text-align: center;
-}
-
-.floating-ask-ai-button svg {
-    width: 24px; /* Control icon size */
-    height: 24px;
-}
-
-.floating-ask-ai-button span {
-    font-size: 0.7em;
-    margin-top: 2px; /* Space between icon and text */
-    display: block; /* Ensure it takes space */
-     line-height: 1;
-}
-
-
-.floating-ask-ai-button:hover {
-    background-color: var(--primary-color, #50ffff);
-    transform: scale(1.05); /* Slight grow effect */
-}
-
-.floating-ask-ai-button:focus {
-     outline: 2px solid var(--primary-color);
-     outline-offset: 2px;
-}
-
-/* Optional: Hide text on smaller screens if needed */
-@media screen and (max-width: 768px) {
-     .floating-ask-ai-button span {
-        /* display: none; */ /* Uncomment to hide text */
-     }
-     .floating-ask-ai-button {
-        width: 55px;
-        height: 55px;
-        bottom: 20px;
-        right: 20px;
-     }
-}

docs/md_v2/assets/selection_ask_ai.js

@@ -1,109 +0,0 @@
-// ==== File: docs/assets/selection_ask_ai.js ====
-
-document.addEventListener('DOMContentLoaded', () => {
-    let askAiButton = null;
-    const askAiPageUrl = '/core/ask-ai/'; // Adjust if your Ask AI page path is different
-
-    function createAskAiButton() {
-        const button = document.createElement('button');
-        button.id = 'ask-ai-selection-btn';
-        button.className = 'ask-ai-selection-button';
-        button.textContent = 'Ask AI'; // Or use an icon
-        button.style.display = 'none'; // Initially hidden
-        button.style.position = 'absolute';
-        button.style.zIndex = '1500'; // Ensure it's on top
-        document.body.appendChild(button);
-
-        button.addEventListener('click', handleAskAiClick);
-        return button;
-    }
-
-    function getSafeSelectedText() {
-        const selection = window.getSelection();
-        if (!selection || selection.rangeCount === 0) {
-            return null;
-        }
-        // Avoid selecting text within the button itself if it was somehow selected
-        const container = selection.getRangeAt(0).commonAncestorContainer;
-        if (askAiButton && askAiButton.contains(container)) {
-             return null;
-        }
-
-        const text = selection.toString().trim();
-        return text.length > 0 ? text : null;
-    }
-
-    function positionButton(event) {
-         const selection = window.getSelection();
-         if (!selection || selection.rangeCount === 0 || selection.isCollapsed) {
-             hideButton();
-             return;
-         }
-
-        const range = selection.getRangeAt(0);
-        const rect = range.getBoundingClientRect();
-
-        // Calculate position: top-right of the selection
-        const scrollX = window.scrollX;
-        const scrollY = window.scrollY;
-        const buttonTop = rect.top + scrollY - askAiButton.offsetHeight - 5; // 5px above
-        const buttonLeft = rect.right + scrollX + 5; // 5px to the right
-
-        askAiButton.style.top = `${buttonTop}px`;
-        askAiButton.style.left = `${buttonLeft}px`;
-        askAiButton.style.display = 'block'; // Show the button
-    }
-
-    function hideButton() {
-        if (askAiButton) {
-            askAiButton.style.display = 'none';
-        }
-    }
-
-    function handleAskAiClick(event) {
-        event.stopPropagation(); // Prevent mousedown from hiding button immediately
-        const selectedText = getSafeSelectedText();
-        if (selectedText) {
-            console.log("Selected Text:", selectedText);
-            // Base64 encode for URL safety (handles special chars, line breaks)
-            // Use encodeURIComponent first for proper Unicode handling before btoa
-            const encodedText = btoa(unescape(encodeURIComponent(selectedText)));
-            const targetUrl = `${askAiPageUrl}?qq=${encodedText}`;
-            console.log("Navigating to:", targetUrl);
-            window.location.href = targetUrl; // Navigate to Ask AI page
-        }
-        hideButton(); // Hide after click
-    }
-
-    // --- Event Listeners ---
-
-    // Show button on mouse up after selection
-    document.addEventListener('mouseup', (event) => {
-        // Slight delay to ensure selection is registered
-        setTimeout(() => {
-            const selectedText = getSafeSelectedText();
-            if (selectedText) {
-                if (!askAiButton) {
-                    askAiButton = createAskAiButton();
-                }
-                // Don't position if the click was ON the button itself
-                if (event.target !== askAiButton) {
-                     positionButton(event);
-                }
-            } else {
-                hideButton();
-            }
-        }, 10); // Small delay
-    });
-
-    // Hide button on scroll or click elsewhere
-    document.addEventListener('mousedown', (event) => {
-        // Hide if clicking anywhere EXCEPT the button itself
-        if (askAiButton && event.target !== askAiButton) {
-            hideButton();
-        }
-    });
-    document.addEventListener('scroll', hideButton, true); // Capture scroll events
-
-    console.log("Selection Ask AI script loaded.");
-});

docs/md_v2/assets/styles.css

@@ -6,8 +6,8 @@
 }
 
 :root {
-    --global-font-size: 14px;
-    --global-code-font-size: 13px;
+    --global-font-size: 16px;
+    --global-code-font-size: 16px;
     --global-line-height: 1.5em;
     --global-space: 10px;
     --font-stack: Menlo, Monaco, Lucida Console, Liberation Mono, DejaVu Sans Mono, Bitstream Vera Sans Mono,
@@ -50,17 +50,8 @@
     --display-h1-decoration: none;
 
     --display-h1-decoration: none;
-
-    --header-height: 65px; /* Adjust based on your actual header height */
-    --sidebar-width: 280px; /* Adjust based on your desired sidebar width */
-    --toc-width: 240px; /* Adjust based on your desired ToC width */
-    --layout-transition-speed: 0.2s; /* For potential future animations */
-
-    --page-width : 100em; /* Adjust based on your design */
 }
 
-
-
 /* body {
     background-color: var(--background-color);
     color: var(--font-color);
@@ -265,6 +256,4 @@ div.badges a {
 }
 div.badges a > img {
     width: auto;
-}
-
-
+}

docs/md_v2/assets/toc.js

@@ -1,144 +0,0 @@
-// ==== File: assets/toc.js ====
-
-document.addEventListener('DOMContentLoaded', () => {
-    const mainContent = document.getElementById('terminal-mkdocs-main-content');
-    const tocContainer = document.getElementById('toc-sidebar');
-    const mainGrid = document.querySelector('.terminal-mkdocs-main-grid'); // Get the flex container
-
-    if (!mainContent) {
-        console.warn("TOC Generator: Main content area '#terminal-mkdocs-main-content' not found.");
-        return;
-    }
-
-    // --- Create ToC container if it doesn't exist ---
-    let tocElement = tocContainer;
-    if (!tocElement) {
-        if (!mainGrid) {
-            console.warn("TOC Generator: Flex container '.terminal-mkdocs-main-grid' not found to append ToC.");
-            return;
-        }
-        tocElement = document.createElement('aside');
-        tocElement.id = 'toc-sidebar';
-        tocElement.style.display = 'none'; // Keep hidden initially
-        // Append it as the last child of the flex grid
-        mainGrid.appendChild(tocElement);
-        console.info("TOC Generator: Created '#toc-sidebar' element.");
-    }
-
-    // --- Find Headings (h2, h3, h4 are common for ToC) ---
-    const headings = mainContent.querySelectorAll('h2, h3, h4');
-    if (headings.length === 0) {
-        console.info("TOC Generator: No headings found on this page. ToC not generated.");
-        tocElement.style.display = 'none'; // Ensure it's hidden
-        return;
-    }
-
-    // --- Generate ToC List ---
-    const tocList = document.createElement('ul');
-    const observerTargets = []; // Store headings for IntersectionObserver
-
-    headings.forEach((heading, index) => {
-        // Ensure heading has an ID for linking
-        if (!heading.id) {
-            // Create a simple slug-like ID
-            heading.id = `toc-heading-${index}-${heading.textContent.toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, '')}`;
-        }
-
-        const listItem = document.createElement('li');
-        const link = document.createElement('a');
-
-        link.href = `#${heading.id}`;
-        link.textContent = heading.textContent;
-
-        // Add class for styling based on heading level
-        const level = parseInt(heading.tagName.substring(1), 10); // Get 2, 3, or 4
-        listItem.classList.add(`toc-level-${level}`);
-
-        listItem.appendChild(link);
-        tocList.appendChild(listItem);
-        observerTargets.push(heading); // Add to observer list
-    });
-
-    // --- Populate and Show ToC ---
-    // Optional: Add a title
-    const tocTitle = document.createElement('h4');
-    tocTitle.textContent = 'On this page'; // Customize title if needed
-
-    tocElement.innerHTML = ''; // Clear previous content if any
-    tocElement.appendChild(tocTitle);
-    tocElement.appendChild(tocList);
-    tocElement.style.display = ''; // Show the ToC container
-
-    console.info(`TOC Generator: Generated ToC with ${headings.length} items.`);
-
-    // --- Scroll Spy using Intersection Observer ---
-    const tocLinks = tocElement.querySelectorAll('a');
-    let activeLink = null; // Keep track of the current active link
-
-    const observerOptions = {
-        // Observe changes relative to the viewport, offset by the header height
-        // Negative top margin pushes the intersection trigger point down
-        // Negative bottom margin ensures elements low on the screen can trigger before they exit
-        rootMargin: `-${getComputedStyle(document.documentElement).getPropertyValue('--header-height').trim()} 0px -60% 0px`,
-        threshold: 0 // Trigger as soon as any part enters/exits the boundary
-    };
-
-    const observerCallback = (entries) => {
-        let topmostVisibleHeading = null;
-
-        entries.forEach(entry => {
-            const link = tocElement.querySelector(`a[href="#${entry.target.id}"]`);
-            if (!link) return;
-
-            // Check if the heading is intersecting (partially or fully visible within rootMargin)
-            if (entry.isIntersecting) {
-                 // Among visible headings, find the one closest to the top edge (within the rootMargin)
-                if (!topmostVisibleHeading || entry.boundingClientRect.top < topmostVisibleHeading.boundingClientRect.top) {
-                    topmostVisibleHeading = entry.target;
-                 }
-            }
-        });
-
-        // If we found a topmost visible heading, activate its link
-        if (topmostVisibleHeading) {
-            const newActiveLink = tocElement.querySelector(`a[href="#${topmostVisibleHeading.id}"]`);
-            if (newActiveLink && newActiveLink !== activeLink) {
-                 // Remove active class from previous link
-                 if (activeLink) {
-                     activeLink.classList.remove('active');
-                     activeLink.parentElement.classList.remove('active-parent'); // Optional parent styling
-                 }
-                 // Add active class to the new link
-                 newActiveLink.classList.add('active');
-                 newActiveLink.parentElement.classList.add('active-parent'); // Optional parent styling
-                 activeLink = newActiveLink;
-
-                 // Optional: Scroll the ToC sidebar to keep the active link visible
-                 // newActiveLink.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
-            }
-        }
-        // If no headings are intersecting (scrolled past the last one?), maybe deactivate all
-        // Or keep the last one active - depends on desired behavior. Current logic keeps last active.
-    };
-
-    const observer = new IntersectionObserver(observerCallback, observerOptions);
-
-    // Observe all target headings
-    observerTargets.forEach(heading => observer.observe(heading));
-
-    // Initial check in case a heading is already in view on load
-    // (Requires slight delay for accurate layout calculation)
-    setTimeout(() => {
-        observerCallback(observer.takeRecords()); // Process initial state
-    }, 100);
-
-    // move footer and the hr before footer to the end of the main content
-    const footer = document.querySelector('footer');
-    const hr = footer.previousElementSibling;
-    if (hr && hr.tagName === 'HR') {
-        mainContent.appendChild(hr);
-    }
-    mainContent.appendChild(footer);
-    console.info("TOC Generator: Footer moved to the end of the main content.");
-
-});

docs/md_v2/core/ask-ai.md

@@ -1,74 +0,0 @@
-<div class="ask-ai-container">
-<iframe id="ask-ai-frame" src="../../ask_ai/index.html" width="100%" style="border:none; display: block;" title="Crawl4AI Assistant"></iframe>
-</div>
-
-<script>
-// Iframe height adjustment
-function resizeAskAiIframe() {
-  const iframe = document.getElementById('ask-ai-frame');
-  if (iframe) {
-    const headerHeight = parseFloat(getComputedStyle(document.documentElement).getPropertyValue('--header-height') || '55');
-    // Footer is removed by JS below, so calculate height based on header + small buffer
-    const topOffset = headerHeight + 20; // Header + buffer/margin
-
-    const availableHeight = window.innerHeight - topOffset;
-    iframe.style.height = Math.max(600, availableHeight) + 'px'; // Min height 600px
-  }
-}
-
-// Run immediately and on resize/load
-resizeAskAiIframe(); // Initial call
-let resizeTimer;
-window.addEventListener('load', resizeAskAiIframe);
-window.addEventListener('resize', () => {
-    clearTimeout(resizeTimer);
-    resizeTimer = setTimeout(resizeAskAiIframe, 150);
-});
-
-// Remove Footer & HR from parent page (DOM Ready might be safer)
-document.addEventListener('DOMContentLoaded', () => {
-    setTimeout(() => { // Add slight delay just in case elements render slowly
-        const footer = window.parent.document.querySelector('footer'); // Target parent document
-        if (footer) {
-            const hrBeforeFooter = footer.previousElementSibling;
-            if (hrBeforeFooter && hrBeforeFooter.tagName === 'HR') {
-                hrBeforeFooter.remove();
-            }
-            footer.remove();
-            // Trigger resize again after removing footer
-            resizeAskAiIframe();
-        } else {
-             console.warn("Ask AI Page: Could not find footer in parent document to remove.");
-        }
-    }, 100); // Shorter delay
-});
-</script>
-
-<style>
-#terminal-mkdocs-main-content {
-    padding: 0 !important;
-    margin: 0;
-    width: 100%;
-    height: 100%;
-    overflow: hidden; /* Prevent body scrollbars, panels handle scroll */
-}
-
-/* Ensure iframe container takes full space */
-#terminal-mkdocs-main-content .ask-ai-container {
-    /* Remove negative margins if footer removal handles space */
-     margin: 0;
-    padding: 0;
-    max-width: none;
-    /* Let the JS set the height */
-    /* height: 600px; Initial fallback height */
-    overflow: hidden; /* Hide potential overflow before JS resize */
-}
-
-/* Hide title/paragraph if they were part of the markdown */
-/* Alternatively, just remove them from the .md file directly */
-/* #terminal-mkdocs-main-content > h1,
-#terminal-mkdocs-main-content > p:first-of-type {
-    display: none;
-} */
-
-</style>

docs/md_v2/core/browser-crawler-config.md

@@ -136,7 +136,6 @@ class CrawlerRunConfig:
         wait_for=None,
         screenshot=False,
         pdf=False,
-        capture_mhtml=False,
         enable_rate_limiting=False,
         rate_limit_config=None,
         memory_threshold_percent=70.0,
@@ -176,9 +175,10 @@ class CrawlerRunConfig:
    - A CSS or JS expression to wait for before extracting content.  
    - Common usage: `wait_for="css:.main-loaded"` or `wait_for="js:() => window.loaded === true"`.
 
-7. **`screenshot`**, **`pdf`**, & **`capture_mhtml`**:  
-   - If `True`, captures a screenshot, PDF, or MHTML snapshot after the page is fully loaded.  
-   - The results go to `result.screenshot` (base64), `result.pdf` (bytes), or `result.mhtml` (string).
+7. **`screenshot`** & **`pdf`**:  
+   - If `True`, captures a screenshot or PDF after the page is fully loaded.  
+   - The results go to `result.screenshot` (base64) or `result.pdf` (bytes).
+
 8. **`verbose`**:  
    - Logs additional runtime details.  
    - Overlaps with the browser’s verbosity if also set to `True` in `BrowserConfig`.

docs/md_v2/core/crawler-result.md

@@ -26,7 +26,6 @@ class CrawlResult(BaseModel):
     downloaded_files: Optional[List[str]] = None
     screenshot: Optional[str] = None
     pdf : Optional[bytes] = None
-    mhtml: Optional[str] = None
     markdown: Optional[Union[str, MarkdownGenerationResult]] = None
     extracted_content: Optional[str] = None
     metadata: Optional[dict] = None
@@ -52,7 +51,6 @@ class CrawlResult(BaseModel):
 | **downloaded_files (`Optional[List[str]]`)** | If `accept_downloads=True` in `BrowserConfig`, this lists the filepaths of saved downloads.         |
 | **screenshot (`Optional[str]`)**          | Screenshot of the page (base64-encoded) if `screenshot=True`.                                       |
 | **pdf (`Optional[bytes]`)**               | PDF of the page if `pdf=True`.                                                                      |
-| **mhtml (`Optional[str]`)**               | MHTML snapshot of the page if `capture_mhtml=True`. Contains the full page with all resources.      |
 | **markdown (`Optional[str or MarkdownGenerationResult]`)** | It holds a `MarkdownGenerationResult`. Over time, this will be consolidated into `markdown`. The generator can provide raw markdown, citations, references, and optionally `fit_markdown`. |
 | **extracted_content (`Optional[str]`)**   | The output of a structured extraction (CSS/LLM-based) stored as JSON string or other text.          |
 | **metadata (`Optional[dict]`)**           | Additional info about the crawl or extracted data.                                                  |
@@ -192,27 +190,18 @@ for img in images:
     print("Image URL:", img["src"], "Alt:", img.get("alt"))
 ```
 
-### 5.3 `screenshot`, `pdf`, and `mhtml`
+### 5.3 `screenshot` and `pdf`
 
-If you set `screenshot=True`, `pdf=True`, or `capture_mhtml=True` in **`CrawlerRunConfig`**, then:
+If you set `screenshot=True` or `pdf=True` in **`CrawlerRunConfig`**, then:
 
-- `result.screenshot` contains a base64-encoded PNG string.
+- `result.screenshot` contains a base64-encoded PNG string.  
 - `result.pdf` contains raw PDF bytes (you can write them to a file).
-- `result.mhtml` contains the MHTML snapshot of the page as a string (you can write it to a .mhtml file).
 
 ```python
-# Save the PDF
 with open("page.pdf", "wb") as f:
     f.write(result.pdf)
-
-# Save the MHTML
-if result.mhtml:
-    with open("page.mhtml", "w", encoding="utf-8") as f:
-        f.write(result.mhtml)
 ```
 
-The MHTML (MIME HTML) format is particularly useful as it captures the entire web page including all of its resources (CSS, images, scripts, etc.) in a single file, making it perfect for archiving or offline viewing.
-
 ### 5.4 `ssl_certificate`
 
 If `fetch_ssl_certificate=True`, `result.ssl_certificate` holds details about the site’s SSL cert, such as issuer, validity dates, etc.

docs/md_v2/core/link-media.md

@@ -4,35 +4,7 @@ In this tutorial, you’ll learn how to:
 
 1. Extract links (internal, external) from crawled pages  
 2. Filter or exclude specific domains (e.g., social media or custom domains)  
-3. Access and ma### 3.2 Excluding Images
-
-#### Excluding External Images
-
-If you're dealing with heavy pages or want to skip third-party images (advertisements, for example), you can turn on:
-
-```python
-crawler_cfg = CrawlerRunConfig(
-    exclude_external_images=True
-)
-```
-
-This setting attempts to discard images from outside the primary domain, keeping only those from the site you're crawling.
-
-#### Excluding All Images
-
-If you want to completely remove all images from the page to maximize performance and reduce memory usage, use:
-
-```python
-crawler_cfg = CrawlerRunConfig(
-    exclude_all_images=True
-)
-```
-
-This setting removes all images very early in the processing pipeline, which significantly improves memory efficiency and processing speed. This is particularly useful when:
-- You don't need image data in your results
-- You're crawling image-heavy pages that cause memory issues
-- You want to focus only on text content
-- You need to maximize crawling speeddata (especially images) in the crawl result  
+3. Access and manage media data (especially images) in the crawl result  
 4. Configure your crawler to exclude or prioritize certain images
 
 > **Prerequisites**  
@@ -299,41 +271,8 @@ Each extracted table contains:
 
 - **`screenshot`**: Set to `True` if you want a full-page screenshot stored as `base64` in `result.screenshot`.  
 - **`pdf`**: Set to `True` if you want a PDF version of the page in `result.pdf`.  
-- **`capture_mhtml`**: Set to `True` if you want an MHTML snapshot of the page in `result.mhtml`. This format preserves the entire web page with all its resources (CSS, images, scripts) in a single file, making it perfect for archiving or offline viewing.
 - **`wait_for_images`**: If `True`, attempts to wait until images are fully loaded before final extraction.
 
-#### Example: Capturing Page as MHTML
-
-```python
-import asyncio
-from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
-
-async def main():
-    crawler_cfg = CrawlerRunConfig(
-        capture_mhtml=True  # Enable MHTML capture
-    )
-    
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun("https://example.com", config=crawler_cfg)
-        
-        if result.success and result.mhtml:
-            # Save the MHTML snapshot to a file
-            with open("example.mhtml", "w", encoding="utf-8") as f:
-                f.write(result.mhtml)
-            print("MHTML snapshot saved to example.mhtml")
-        else:
-            print("Failed to capture MHTML:", result.error_message)
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-The MHTML format is particularly useful because:
-- It captures the complete page state including all resources
-- It can be opened in most modern browsers for offline viewing
-- It preserves the page exactly as it appeared during crawling
-- It's a single file, making it easy to store and transfer
-
 ---
 
 ## 4. Putting It All Together: Link & Media Filtering

mkdocs.yml

@@ -7,11 +7,10 @@ docs_dir: docs/md_v2
 
 nav:
   - Home: 'index.md'
-  - "Ask AI": "core/ask-ai.md"
-  - "Quick Start": "core/quickstart.md"
   - Setup & Installation:
     - "Installation": "core/installation.md"
     - "Docker Deployment": "core/docker-deployment.md"
+  - "Quick Start": "core/quickstart.md"
   - "Blog & Changelog":
     - "Blog Home": "blog/index.md"
     - "Changelog": "https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md"
@@ -39,7 +38,6 @@ nav:
     - "Crawl Dispatcher": "advanced/crawl-dispatcher.md"
     - "Identity Based Crawling": "advanced/identity-based-crawling.md"
     - "SSL Certificate": "advanced/ssl-certificate.md"
-    - "Network & Console Capture": "advanced/network-console-capture.md"
   - Extraction:
     - "LLM-Free Strategies": "extraction/no-llm-strategies.md"
     - "LLM Strategies": "extraction/llm-strategies.md"
@@ -77,7 +75,6 @@ extra:
   version: !ENV [CRAWL4AI_VERSION, 'development']
 
 extra_css:
-  - assets/layout.css
   - assets/styles.css
   - assets/highlight.css
   - assets/dmvendor.css
@@ -85,9 +82,4 @@ extra_css:
 extra_javascript:
   - assets/highlight.min.js
   - assets/highlight_init.js
-  - https://buttons.github.io/buttons.js
-  - assets/toc.js
-  - assets/github_stats.js 
-  - assets/selection_ask_ai.js
-  - assets/copy_code.js
-  - assets/floating_ask_ai_button.js
+  - https://buttons.github.io/buttons.js

parameter_updates.txt

@@ -1,20 +0,0 @@
-The file /docs/md_v2/api/parameters.md should be updated to include the new network and console capturing parameters. 
-
-Here's what needs to be updated:
-
-1. Change section title from:
-```
-### G) **Debug & Logging**
-```
-to:
-```
-### G) **Debug, Logging & Capturing**
-```
-
-2. Add new parameters to the table:
-```
-| **`capture_network_requests`** | `bool` (False) | Captures all network requests, responses, and failures during the crawl. Available in `result.network_requests`. |
-| **`capture_console_messages`** | `bool` (False) | Captures all browser console messages (logs, warnings, errors) during the crawl. Available in `result.console_messages`. |
-```
-
-These changes demonstrate how to use the new network and console capturing features in the CrawlerRunConfig.

prompts/prompt_net_requests.md

@@ -1,489 +0,0 @@
-I want to enhance the `AsyncPlaywrightCrawlerStrategy` to optionally capture network requests and console messages during a crawl, storing them in the final `CrawlResult`.
-
-Here's a breakdown of the proposed changes across the relevant files:
-
-**1. Configuration (`crawl4ai/async_configs.py`)**
-
-*   **Goal:** Add flags to `CrawlerRunConfig` to enable/disable capturing.
-*   **Changes:**
-    *   Add two new boolean attributes to `CrawlerRunConfig`:
-        *   `capture_network_requests: bool = False`
-        *   `capture_console_messages: bool = False`
-    *   Update `__init__`, `from_kwargs`, `to_dict`, and implicitly `clone`/`dump`/`load` to include these new attributes.
-
-```python
-# ==== File: crawl4ai/async_configs.py ====
-# ... (imports) ...
-
-class CrawlerRunConfig():
-    # ... (existing attributes) ...
-
-    # NEW: Network and Console Capturing Parameters
-    capture_network_requests: bool = False
-    capture_console_messages: bool = False
-
-    # Experimental Parameters
-    experimental: Dict[str, Any] = None,
-
-    def __init__(
-        self,
-        # ... (existing parameters) ...
-
-        # NEW: Network and Console Capturing Parameters
-        capture_network_requests: bool = False,
-        capture_console_messages: bool = False,
-
-        # Experimental Parameters
-        experimental: Dict[str, Any] = None,
-    ):
-        # ... (existing assignments) ...
-
-        # NEW: Assign new parameters
-        self.capture_network_requests = capture_network_requests
-        self.capture_console_messages = capture_console_messages
-
-        # Experimental Parameters
-        self.experimental = experimental or {}
-
-        # ... (rest of __init__) ...
-
-    @staticmethod
-    def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
-        return CrawlerRunConfig(
-            # ... (existing kwargs gets) ...
-
-            # NEW: Get new parameters
-            capture_network_requests=kwargs.get("capture_network_requests", False),
-            capture_console_messages=kwargs.get("capture_console_messages", False),
-
-            # Experimental Parameters
-            experimental=kwargs.get("experimental"),
-        )
-
-    def to_dict(self):
-        return {
-            # ... (existing dict entries) ...
-
-            # NEW: Add new parameters to dict
-            "capture_network_requests": self.capture_network_requests,
-            "capture_console_messages": self.capture_console_messages,
-
-            "experimental": self.experimental,
-        }
-
-    # clone(), dump(), load() should work automatically if they rely on to_dict() and from_kwargs()
-    # or the serialization logic correctly handles all attributes.
-```
-
-**2. Data Models (`crawl4ai/models.py`)**
-
-*   **Goal:** Add fields to store the captured data in the response/result objects.
-*   **Changes:**
-    *   Add `network_requests: Optional[List[Dict[str, Any]]] = None` and `console_messages: Optional[List[Dict[str, Any]]] = None` to `AsyncCrawlResponse`.
-    *   Add the same fields to `CrawlResult`.
-
-```python
-# ==== File: crawl4ai/models.py ====
-# ... (imports) ...
-
-# ... (Existing dataclasses/models) ...
-
-class AsyncCrawlResponse(BaseModel):
-    html: str
-    response_headers: Dict[str, str]
-    js_execution_result: Optional[Dict[str, Any]] = None
-    status_code: int
-    screenshot: Optional[str] = None
-    pdf_data: Optional[bytes] = None
-    get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
-    downloaded_files: Optional[List[str]] = None
-    ssl_certificate: Optional[SSLCertificate] = None
-    redirected_url: Optional[str] = None
-    # NEW: Fields for captured data
-    network_requests: Optional[List[Dict[str, Any]]] = None
-    console_messages: Optional[List[Dict[str, Any]]] = None
-
-    class Config:
-        arbitrary_types_allowed = True
-
-# ... (Existing models like MediaItem, Link, etc.) ...
-
-class CrawlResult(BaseModel):
-    url: str
-    html: str
-    success: bool
-    cleaned_html: Optional[str] = None
-    media: Dict[str, List[Dict]] = {}
-    links: Dict[str, List[Dict]] = {}
-    downloaded_files: Optional[List[str]] = None
-    js_execution_result: Optional[Dict[str, Any]] = None
-    screenshot: Optional[str] = None
-    pdf: Optional[bytes] = None
-    mhtml: Optional[str] = None # Added mhtml based on the provided models.py
-    _markdown: Optional[MarkdownGenerationResult] = PrivateAttr(default=None)
-    extracted_content: Optional[str] = None
-    metadata: Optional[dict] = None
-    error_message: Optional[str] = None
-    session_id: Optional[str] = None
-    response_headers: Optional[dict] = None
-    status_code: Optional[int] = None
-    ssl_certificate: Optional[SSLCertificate] = None
-    dispatch_result: Optional[DispatchResult] = None
-    redirected_url: Optional[str] = None
-    # NEW: Fields for captured data
-    network_requests: Optional[List[Dict[str, Any]]] = None
-    console_messages: Optional[List[Dict[str, Any]]] = None
-
-    class Config:
-        arbitrary_types_allowed = True
-
-    # ... (Existing __init__, properties, model_dump for markdown compatibility) ...
-
-# ... (Rest of the models) ...
-```
-
-**3. Crawler Strategy (`crawl4ai/async_crawler_strategy.py`)**
-
-*   **Goal:** Implement the actual capturing logic within `AsyncPlaywrightCrawlerStrategy._crawl_web`.
-*   **Changes:**
-    *   Inside `_crawl_web`, initialize empty lists `captured_requests = []` and `captured_console = []`.
-    *   Conditionally attach Playwright event listeners (`page.on(...)`) based on the `config.capture_network_requests` and `config.capture_console_messages` flags.
-    *   Define handler functions for these listeners to extract relevant data and append it to the respective lists. Include timestamps.
-    *   Pass the captured lists to the `AsyncCrawlResponse` constructor at the end of the method.
-
-```python
-# ==== File: crawl4ai/async_crawler_strategy.py ====
-# ... (imports) ...
-import time # Make sure time is imported
-
-class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
-    # ... (existing methods like __init__, start, close, etc.) ...
-
-    async def _crawl_web(
-        self, url: str, config: CrawlerRunConfig
-    ) -> AsyncCrawlResponse:
-        """
-        Internal method to crawl web URLs with the specified configuration.
-        Includes optional network and console capturing. # MODIFIED DOCSTRING
-        """
-        config.url = url
-        response_headers = {}
-        execution_result = None
-        status_code = None
-        redirected_url = url
-
-        # Reset downloaded files list for new crawl
-        self._downloaded_files = []
-
-        # Initialize capture lists - IMPORTANT: Reset per crawl
-        captured_requests: List[Dict[str, Any]] = []
-        captured_console: List[Dict[str, Any]] = []
-
-        # Handle user agent ... (existing code) ...
-
-        # Get page for session
-        page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
-
-        # ... (existing code for cookies, navigator overrides, hooks) ...
-
-        # --- Setup Capturing Listeners ---
-        # NOTE: These listeners are attached *before* page.goto()
-
-        # Network Request Capturing
-        if config.capture_network_requests:
-            async def handle_request_capture(request):
-                try:
-                    post_data_str = None
-                    try:
-                        # Be cautious with large post data
-                        post_data = request.post_data_buffer
-                        if post_data:
-                             # Attempt to decode, fallback to base64 or size indication
-                             try:
-                                 post_data_str = post_data.decode('utf-8', errors='replace')
-                             except UnicodeDecodeError:
-                                 post_data_str = f"[Binary data: {len(post_data)} bytes]"
-                    except Exception:
-                        post_data_str = "[Error retrieving post data]"
-
-                    captured_requests.append({
-                        "event_type": "request",
-                        "url": request.url,
-                        "method": request.method,
-                        "headers": dict(request.headers), # Convert Header dict
-                        "post_data": post_data_str,
-                        "resource_type": request.resource_type,
-                        "is_navigation_request": request.is_navigation_request(),
-                        "timestamp": time.time()
-                    })
-                except Exception as e:
-                    self.logger.warning(f"Error capturing request details for {request.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "request_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()})
-
-            async def handle_response_capture(response):
-                try:
-                    # Avoid capturing full response body by default due to size/security
-                    # security_details = await response.security_details() # Optional: More SSL info
-                    captured_requests.append({
-                        "event_type": "response",
-                        "url": response.url,
-                        "status": response.status,
-                        "status_text": response.status_text,
-                        "headers": dict(response.headers), # Convert Header dict
-                        "from_service_worker": response.from_service_worker,
-                        # "security_details": security_details, # Uncomment if needed
-                        "request_timing": response.request.timing, # Detailed timing info
-                        "timestamp": time.time()
-                    })
-                except Exception as e:
-                    self.logger.warning(f"Error capturing response details for {response.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "response_capture_error", "url": response.url, "error": str(e), "timestamp": time.time()})
-
-            async def handle_request_failed_capture(request):
-                 try:
-                    captured_requests.append({
-                        "event_type": "request_failed",
-                        "url": request.url,
-                        "method": request.method,
-                        "resource_type": request.resource_type,
-                        "failure_text": request.failure.error_text if request.failure else "Unknown failure",
-                        "timestamp": time.time()
-                    })
-                 except Exception as e:
-                    self.logger.warning(f"Error capturing request failed details for {request.url}: {e}", tag="CAPTURE")
-                    captured_requests.append({"event_type": "request_failed_capture_error", "url": request.url, "error": str(e), "timestamp": time.time()})
-
-            page.on("request", handle_request_capture)
-            page.on("response", handle_response_capture)
-            page.on("requestfailed", handle_request_failed_capture)
-
-        # Console Message Capturing
-        if config.capture_console_messages:
-            def handle_console_capture(msg):
-                 try:
-                    location = msg.location()
-                    # Attempt to resolve JSHandle args to primitive values
-                    resolved_args = []
-                    try:
-                        for arg in msg.args:
-                            resolved_args.append(arg.json_value()) # May fail for complex objects
-                    except Exception:
-                         resolved_args.append("[Could not resolve JSHandle args]")
-
-                    captured_console.append({
-                        "type": msg.type(), # e.g., 'log', 'error', 'warning'
-                        "text": msg.text(),
-                        "args": resolved_args, # Captured arguments
-                        "location": f"{location['url']}:{location['lineNumber']}:{location['columnNumber']}" if location else "N/A",
-                        "timestamp": time.time()
-                    })
-                 except Exception as e:
-                    self.logger.warning(f"Error capturing console message: {e}", tag="CAPTURE")
-                    captured_console.append({"type": "console_capture_error", "error": str(e), "timestamp": time.time()})
-
-            def handle_pageerror_capture(err):
-                 try:
-                    captured_console.append({
-                        "type": "error", # Consistent type for page errors
-                        "text": err.message,
-                        "stack": err.stack,
-                        "timestamp": time.time()
-                    })
-                 except Exception as e:
-                    self.logger.warning(f"Error capturing page error: {e}", tag="CAPTURE")
-                    captured_console.append({"type": "pageerror_capture_error", "error": str(e), "timestamp": time.time()})
-
-            page.on("console", handle_console_capture)
-            page.on("pageerror", handle_pageerror_capture)
-        # --- End Setup Capturing Listeners ---
-
-
-        # Set up console logging if requested (Keep original logging logic separate or merge carefully)
-        if config.log_console:
-            # ... (original log_console setup using page.on(...) remains here) ...
-            # This allows logging to screen *and* capturing to the list if both flags are True
-            def log_consol(msg, console_log_type="debug"):
-                # ... existing implementation ...
-                pass # Placeholder for existing code
-
-            page.on("console", lambda msg: log_consol(msg, "debug"))
-            page.on("pageerror", lambda e: log_consol(e, "error"))
-
-
-        try:
-            # ... (existing code for SSL, downloads, goto, waits, JS execution, etc.) ...
-
-            # Get final HTML content
-            # ... (existing code for selector logic or page.content()) ...
-            if config.css_selector:
-                # ... existing selector logic ...
-                html = f"<div class='crawl4ai-result'>\n" + "\n".join(html_parts) + "\n</div>"
-            else:
-                html = await page.content()
-
-            await self.execute_hook(
-                "before_return_html", page=page, html=html, context=context, config=config
-            )
-
-            # Handle PDF and screenshot generation
-            # ... (existing code) ...
-
-            # Define delayed content getter
-            # ... (existing code) ...
-
-            # Return complete response - ADD CAPTURED DATA HERE
-            return AsyncCrawlResponse(
-                html=html,
-                response_headers=response_headers,
-                js_execution_result=execution_result,
-                status_code=status_code,
-                screenshot=screenshot_data,
-                pdf_data=pdf_data,
-                get_delayed_content=get_delayed_content,
-                ssl_certificate=ssl_cert,
-                downloaded_files=(
-                    self._downloaded_files if self._downloaded_files else None
-                ),
-                redirected_url=redirected_url,
-                # NEW: Pass captured data conditionally
-                network_requests=captured_requests if config.capture_network_requests else None,
-                console_messages=captured_console if config.capture_console_messages else None,
-            )
-
-        except Exception as e:
-            raise e # Re-raise the original exception
-
-        finally:
-            # If no session_id is given we should close the page
-            if not config.session_id:
-                # Detach listeners before closing to prevent potential errors during close
-                if config.capture_network_requests:
-                    page.remove_listener("request", handle_request_capture)
-                    page.remove_listener("response", handle_response_capture)
-                    page.remove_listener("requestfailed", handle_request_failed_capture)
-                if config.capture_console_messages:
-                    page.remove_listener("console", handle_console_capture)
-                    page.remove_listener("pageerror", handle_pageerror_capture)
-                # Also remove logging listeners if they were attached
-                if config.log_console:
-                    # Need to figure out how to remove the lambdas if necessary,
-                    # or ensure they don't cause issues on close. Often, it's fine.
-                    pass
-
-                await page.close()
-
-    # ... (rest of AsyncPlaywrightCrawlerStrategy methods) ...
-
-```
-
-**4. Core Crawler (`crawl4ai/async_webcrawler.py`)**
-
-*   **Goal:** Ensure the captured data from `AsyncCrawlResponse` is transferred to the final `CrawlResult`.
-*   **Changes:**
-    *   In `arun`, when processing a non-cached result (inside the `if not cached_result or not html:` block), after receiving `async_response` and calling `aprocess_html` to get `crawl_result`, copy the `network_requests` and `console_messages` from `async_response` to `crawl_result`.
-
-```python
-# ==== File: crawl4ai/async_webcrawler.py ====
-# ... (imports) ...
-
-class AsyncWebCrawler:
-    # ... (existing methods) ...
-
-    async def arun(
-        self,
-        url: str,
-        config: CrawlerRunConfig = None,
-        **kwargs,
-    ) -> RunManyReturn:
-        # ... (existing setup, cache check) ...
-
-        async with self._lock or self.nullcontext():
-            try:
-                # ... (existing logging, cache context setup) ...
-
-                if cached_result:
-                    # ... (existing cache handling logic) ...
-                    # Note: Captured network/console usually not useful from cache
-                    # Ensure they are None or empty if read from cache, unless stored explicitly
-                    cached_result.network_requests = cached_result.network_requests or None
-                    cached_result.console_messages = cached_result.console_messages or None
-                    # ... (rest of cache logic) ...
-
-                # Fetch fresh content if needed
-                if not cached_result or not html:
-                    t1 = time.perf_counter()
-
-                    # ... (existing user agent update, robots.txt check) ...
-
-                    ##############################
-                    # Call CrawlerStrategy.crawl #
-                    ##############################
-                    async_response = await self.crawler_strategy.crawl(
-                        url,
-                        config=config,
-                    )
-
-                    # ... (existing assignment of html, screenshot, pdf, js_result from async_response) ...
-
-                    t2 = time.perf_counter()
-                    # ... (existing logging) ...
-
-                    ###############################################################
-                    # Process the HTML content, Call CrawlerStrategy.process_html #
-                    ###############################################################
-                    crawl_result: CrawlResult = await self.aprocess_html(
-                        # ... (existing args) ...
-                    )
-
-                    # --- Transfer data from AsyncCrawlResponse to CrawlResult ---
-                    crawl_result.status_code = async_response.status_code
-                    crawl_result.redirected_url = async_response.redirected_url or url
-                    crawl_result.response_headers = async_response.response_headers
-                    crawl_result.downloaded_files = async_response.downloaded_files
-                    crawl_result.js_execution_result = js_execution_result
-                    crawl_result.ssl_certificate = async_response.ssl_certificate
-                    # NEW: Copy captured data
-                    crawl_result.network_requests = async_response.network_requests
-                    crawl_result.console_messages = async_response.console_messages
-                    # ------------------------------------------------------------
-
-                    crawl_result.success = bool(html)
-                    crawl_result.session_id = getattr(config, "session_id", None)
-
-                    # ... (existing logging) ...
-
-                    # Update cache if appropriate
-                    if cache_context.should_write() and not bool(cached_result):
-                        # crawl_result now includes network/console data if captured
-                        await async_db_manager.acache_url(crawl_result)
-
-                    return CrawlResultContainer(crawl_result)
-
-                else: # Cached result was used
-                     # ... (existing logging for cache hit) ...
-                    cached_result.success = bool(html)
-                    cached_result.session_id = getattr(config, "session_id", None)
-                    cached_result.redirected_url = cached_result.redirected_url or url
-                    return CrawlResultContainer(cached_result)
-
-            except Exception as e:
-                # ... (existing error handling) ...
-                return CrawlResultContainer(
-                    CrawlResult(
-                        url=url, html="", success=False, error_message=error_message
-                    )
-                )
-
-    # ... (aprocess_html remains unchanged regarding capture) ...
-
-    # ... (arun_many remains unchanged regarding capture) ...
-```
-
-**Summary of Changes:**
-
-1.  **Configuration:** Added `capture_network_requests` and `capture_console_messages` flags to `CrawlerRunConfig`.
-2.  **Models:** Added corresponding `network_requests` and `console_messages` fields (List of Dicts) to `AsyncCrawlResponse` and `CrawlResult`.
-3.  **Strategy:** Implemented conditional event listeners in `AsyncPlaywrightCrawlerStrategy._crawl_web` to capture data into lists when flags are true. Populated these fields in the returned `AsyncCrawlResponse`. Added basic error handling within capture handlers. Added timestamps.
-4.  **Crawler:** Modified `AsyncWebCrawler.arun` to copy the captured data from `AsyncCrawlResponse` into the final `CrawlResult` for non-cached fetches.
-
-This approach keeps the capturing logic contained within the Playwright strategy, uses clear configuration flags, and integrates the results into the existing data flow. The data format (list of dictionaries) is flexible for storing varied information from requests/responses/console messages.

tests/browser/docker/test_docker_browser.py

@@ -9,6 +9,8 @@ import os
 import sys
 import shutil
 import uuid
+import json
+from typing import List, Dict, Any, Optional, Tuple
 
 # Add the project root to Python path if running directly
 if __name__ == "__main__":
@@ -17,9 +19,9 @@ if __name__ == "__main__":
 from crawl4ai.browser import BrowserManager
 from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
 from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.browser import DockerConfig
-from crawl4ai.browser import DockerRegistry
-from crawl4ai.browser import DockerUtils
+from crawl4ai.browser.docker_config import DockerConfig
+from crawl4ai.browser.docker_registry import DockerRegistry
+from crawl4ai.browser.docker_utils import DockerUtils
 
 # Create a logger for clear terminal output
 logger = AsyncLogger(verbose=True, log_file=None)
@@ -136,7 +138,7 @@ async def test_docker_components():
             
         # Verify Chrome is installed in the container
         returncode, stdout, stderr = await docker_utils.exec_in_container(
-            container_id, ["which", "chromium"]
+            container_id, ["which", "google-chrome"]
         )
         
         if returncode != 0:
@@ -149,7 +151,7 @@ async def test_docker_components():
         
         # Test Chrome version
         returncode, stdout, stderr = await docker_utils.exec_in_container(
-            container_id, ["chromium", "--version"]
+            container_id, ["google-chrome", "--version"]
         )
         
         if returncode != 0:
@@ -530,7 +532,7 @@ async def test_docker_registry_reuse():
         logger.info("First browser started successfully", tag="TEST")
         
         # Get container ID from the strategy
-        docker_strategy1 = manager1.strategy
+        docker_strategy1 = manager1._strategy
         container_id1 = docker_strategy1.container_id
         logger.info(f"First browser container ID: {container_id1[:12]}", tag="TEST")
         
@@ -560,7 +562,7 @@ async def test_docker_registry_reuse():
         logger.info("Second browser started successfully", tag="TEST")
         
         # Get container ID from the second strategy
-        docker_strategy2 = manager2.strategy
+        docker_strategy2 = manager2._strategy
         container_id2 = docker_strategy2.container_id
         logger.info(f"Second browser container ID: {container_id2[:12]}", tag="TEST")
         
@@ -608,10 +610,10 @@ async def run_tests():
         return
     
     # First test Docker components
-    # setup_result = await test_docker_components()
-    # if not setup_result:
-    #     logger.error("Docker component tests failed - skipping browser tests", tag="TEST")
-    #     return
+    setup_result = await test_docker_components()
+    if not setup_result:
+        logger.error("Docker component tests failed - skipping browser tests", tag="TEST")
+        return
     
     # Run browser tests
     results.append(await test_docker_connect_mode())

tests/browser/manager/demo_browser_manager.py

@@ -1,525 +0,0 @@
-"""Demo script for testing the enhanced BrowserManager.
-
-This script demonstrates the browser pooling capabilities of the enhanced
-BrowserManager with various configurations and usage patterns.
-"""
-
-import asyncio
-import time
-import random
-
-from crawl4ai.browser.manager import BrowserManager, UnavailableBehavior
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-
-import playwright
-
-SAFE_URLS = [
-    "https://example.com",
-    "https://example.com/page1",
-    "https://httpbin.org/get",
-    "https://httpbin.org/html",
-    "https://httpbin.org/ip",
-    "https://httpbin.org/user-agent",
-    "https://httpbin.org/headers",
-    "https://httpbin.org/cookies",
-    "https://httpstat.us/200",
-    "https://httpstat.us/301",
-    "https://httpstat.us/404",
-    "https://httpstat.us/500",
-    "https://jsonplaceholder.typicode.com/posts/1",
-    "https://jsonplaceholder.typicode.com/posts/2",
-    "https://jsonplaceholder.typicode.com/posts/3",
-    "https://jsonplaceholder.typicode.com/posts/4",
-    "https://jsonplaceholder.typicode.com/posts/5",
-    "https://jsonplaceholder.typicode.com/comments/1",
-    "https://jsonplaceholder.typicode.com/comments/2",
-    "https://jsonplaceholder.typicode.com/users/1",
-    "https://jsonplaceholder.typicode.com/users/2",
-    "https://jsonplaceholder.typicode.com/albums/1",
-    "https://jsonplaceholder.typicode.com/albums/2",
-    "https://jsonplaceholder.typicode.com/photos/1",
-    "https://jsonplaceholder.typicode.com/photos/2",
-    "https://jsonplaceholder.typicode.com/todos/1",
-    "https://jsonplaceholder.typicode.com/todos/2",
-    "https://www.iana.org",
-    "https://www.iana.org/domains",
-    "https://www.iana.org/numbers",
-    "https://www.iana.org/protocols",
-    "https://www.iana.org/about",
-    "https://www.iana.org/time-zones",
-    "https://www.data.gov",
-    "https://catalog.data.gov/dataset",
-    "https://www.archives.gov",
-    "https://www.usa.gov",
-    "https://www.loc.gov",
-    "https://www.irs.gov",
-    "https://www.census.gov",
-    "https://www.bls.gov",
-    "https://www.gpo.gov",
-    "https://www.w3.org",
-    "https://www.w3.org/standards",
-    "https://www.w3.org/WAI",
-    "https://www.rfc-editor.org",
-    "https://www.ietf.org",
-    "https://www.icann.org",
-    "https://www.internetsociety.org",
-    "https://www.python.org"
-]
-
-async def basic_pooling_demo():
-    """Demonstrate basic browser pooling functionality."""
-    print("\n=== Basic Browser Pooling Demo ===")
-    
-    # Create logger
-    logger = AsyncLogger(verbose=True)
-    
-    # Create browser configurations
-    config1 = BrowserConfig(
-        browser_type="chromium",
-        headless=True,
-        browser_mode="playwright"
-    )
-    
-    config2 = BrowserConfig(
-        browser_type="chromium", 
-        headless=True,
-        browser_mode="cdp"
-    )
-    
-    # Create browser manager with on-demand behavior
-    manager = BrowserManager(
-        browser_config=config1,
-        logger=logger,
-        unavailable_behavior=UnavailableBehavior.ON_DEMAND,
-        max_browsers_per_config=3
-    )
-    
-    try:
-        # Initialize pool with both configurations
-        print("Initializing browser pool...")
-        await manager.initialize_pool(
-            browser_configs=[config1, config2],
-            browsers_per_config=2
-        )
-        
-        # Display initial pool status
-        status = await manager.get_pool_status()
-        print(f"Initial pool status: {status}")
-        
-        # Create crawler run configurations
-        run_config1 = CrawlerRunConfig()
-        run_config2 = CrawlerRunConfig()
-        
-        # Simulate concurrent page requests
-        print("\nGetting pages for parallel crawling...")
-        
-        # Function to simulate crawling
-        async def simulate_crawl(index: int, config: BrowserConfig, run_config: CrawlerRunConfig):
-            print(f"Crawler {index}: Requesting page...")
-            page, context, strategy = await manager.get_page(run_config, config)
-            print(f"Crawler {index}: Got page, navigating to example.com...")
-            
-            try:
-                await page.goto("https://example.com")
-                title = await page.title()
-                print(f"Crawler {index}: Page title: {title}")
-                
-                # Simulate work
-                await asyncio.sleep(random.uniform(1, 3))
-                print(f"Crawler {index}: Work completed, releasing page...")
-                
-                # Check dynamic page content
-                content = await page.content()
-                content_length = len(content)
-                print(f"Crawler {index}: Page content length: {content_length}")
-                
-            except Exception as e:
-                print(f"Crawler {index}: Error: {str(e)}")
-            finally:
-                # Release the page
-                await manager.release_page(page, strategy, config)
-                print(f"Crawler {index}: Page released")
-        
-        # Create 5 parallel crawls
-        crawl_tasks = []
-        for i in range(5):
-            # Alternate between configurations
-            config = config1 if i % 2 == 0 else config2
-            run_config = run_config1 if i % 2 == 0 else run_config2
-            
-            task = asyncio.create_task(simulate_crawl(i+1, config, run_config))
-            crawl_tasks.append(task)
-        
-        # Wait for all crawls to complete
-        await asyncio.gather(*crawl_tasks)
-        
-        # Display final pool status
-        status = await manager.get_pool_status()
-        print(f"\nFinal pool status: {status}")
-        
-    finally:
-        # Clean up
-        print("\nClosing browser manager...")
-        await manager.close()
-        print("Browser manager closed")
-
-
-async def prewarm_pages_demo():
-    """Demonstrate page pre-warming functionality."""
-    print("\n=== Page Pre-warming Demo ===")
-    
-    # Create logger
-    logger = AsyncLogger(verbose=True)
-    
-    # Create browser configuration
-    config = BrowserConfig(
-        browser_type="chromium",
-        headless=True,
-        browser_mode="playwright"
-    )
-    
-    # Create crawler run configurations for pre-warming
-    run_config1 = CrawlerRunConfig(
-        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    )
-    
-    run_config2 = CrawlerRunConfig(
-        user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15"
-    )
-    
-    # Create page pre-warm configurations
-    page_configs = [
-        (config, run_config1, 2),  # 2 pages with run_config1
-        (config, run_config2, 3)   # 3 pages with run_config2
-    ]
-    
-    # Create browser manager
-    manager = BrowserManager(
-        browser_config=config,
-        logger=logger,
-        unavailable_behavior=UnavailableBehavior.EXCEPTION
-    )
-    
-    try:
-        # Initialize pool with pre-warmed pages
-        print("Initializing browser pool with pre-warmed pages...")
-        await manager.initialize_pool(
-            browser_configs=[config],
-            browsers_per_config=2,
-            page_configs=page_configs
-        )
-        
-        # Display pool status
-        status = await manager.get_pool_status()
-        print(f"Pool status after pre-warming: {status}")
-        
-        # Simulate using pre-warmed pages
-        print("\nUsing pre-warmed pages...")
-        
-        async def use_prewarm_page(index: int, run_config: CrawlerRunConfig):
-            print(f"Task {index}: Requesting pre-warmed page...")
-            page, context, strategy = await manager.get_page(run_config, config)
-            
-            try:
-                print(f"Task {index}: Got page, navigating to example.com...")
-                await page.goto("https://example.com")
-                
-                # Verify user agent was applied correctly
-                user_agent = await page.evaluate("() => navigator.userAgent")
-                print(f"Task {index}: User agent: {user_agent}")
-                
-                # Get page title
-                title = await page.title()
-                print(f"Task {index}: Page title: {title}")
-                
-                # Simulate work
-                await asyncio.sleep(1)
-            finally:
-                # Release the page
-                print(f"Task {index}: Releasing page...")
-                await manager.release_page(page, strategy, config)
-        
-        # Create tasks to use pre-warmed pages
-        tasks = []
-        # Use run_config1 pages
-        for i in range(2):
-            tasks.append(asyncio.create_task(use_prewarm_page(i+1, run_config1)))
-        
-        # Use run_config2 pages
-        for i in range(3):
-            tasks.append(asyncio.create_task(use_prewarm_page(i+3, run_config2)))
-        
-        # Wait for all tasks to complete
-        await asyncio.gather(*tasks)
-        
-        # Try to use more pages than we pre-warmed (should raise exception)
-        print("\nTrying to use more pages than pre-warmed...")
-        try:
-            page, context, strategy = await manager.get_page(run_config1, config)
-            try:
-                print("Got extra page (unexpected)")
-                await page.goto("https://example.com")
-            finally:
-                await manager.release_page(page, strategy, config)
-        except Exception as e:
-            print(f"Expected exception when requesting more pages: {str(e)}")
-        
-    finally:
-        # Clean up
-        print("\nClosing browser manager...")
-        await manager.close()
-        print("Browser manager closed")
-
-
-async def prewarm_on_demand_demo():
-    """Demonstrate pre-warming with on-demand browser creation."""
-    print("\n=== Pre-warming with On-Demand Browser Creation Demo ===")
-    
-    # Create logger
-    logger = AsyncLogger(verbose=True)
-    
-    # Create browser configuration
-    config = BrowserConfig(
-        browser_type="chromium",
-        headless=True,
-        browser_mode="playwright"
-    )
-    
-    # Create crawler run configurations
-    run_config = CrawlerRunConfig(
-        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    )
-    
-    # Create page pre-warm configurations - just pre-warm 2 pages
-    page_configs = [
-        (config, run_config, 2)
-    ]
-    
-    # Create browser manager with ON_DEMAND behavior
-    manager = BrowserManager(
-        browser_config=config,
-        logger=logger,
-        unavailable_behavior=UnavailableBehavior.ON_DEMAND,
-        max_browsers_per_config=5  # Allow up to 5 browsers
-    )
-    
-    try:
-        # Initialize pool with pre-warmed pages
-        print("Initializing browser pool with pre-warmed pages...")
-        await manager.initialize_pool(
-            browser_configs=[config],
-            browsers_per_config=1,  # Start with just 1 browser
-            page_configs=page_configs
-        )
-        
-        # Display initial pool status
-        status = await manager.get_pool_status()
-        print(f"Initial pool status: {status}")
-        
-        # Simulate using more pages than pre-warmed - should create browsers on demand
-        print("\nUsing more pages than pre-warmed (should create on demand)...")
-        
-        async def use_page(index: int):
-            print(f"Task {index}: Requesting page...")
-            page, context, strategy = await manager.get_page(run_config, config)
-            
-            try:
-                print(f"Task {index}: Got page, navigating to example.com...")
-                await page.goto("https://example.com")
-                
-                # Get page title
-                title = await page.title()
-                print(f"Task {index}: Page title: {title}")
-                
-                # Simulate work for a varying amount of time
-                work_time = 1 + (index * 0.5)  # Stagger completion times
-                print(f"Task {index}: Working for {work_time} seconds...")
-                await asyncio.sleep(work_time)
-                print(f"Task {index}: Work completed")
-            finally:
-                # Release the page
-                print(f"Task {index}: Releasing page...")
-                await manager.release_page(page, strategy, config)
-        
-        # Create more tasks than pre-warmed pages
-        tasks = []
-        for i in range(5):  # Try to use 5 pages when only 2 are pre-warmed
-            tasks.append(asyncio.create_task(use_page(i+1)))
-        
-        # Wait for all tasks to complete
-        await asyncio.gather(*tasks)
-        
-        # Display final pool status - should show on-demand created browsers
-        status = await manager.get_pool_status()
-        print(f"\nFinal pool status: {status}")
-        
-    finally:
-        # Clean up
-        print("\nClosing browser manager...")
-        await manager.close()
-        print("Browser manager closed")
-
-
-async def high_volume_demo():
-    """Demonstrate high-volume access to pre-warmed pages."""
-    print("\n=== High Volume Pre-warmed Pages Demo ===")
-    
-    # Create logger
-    logger = AsyncLogger(verbose=True)
-    
-    # Create browser configuration
-    config = BrowserConfig(
-        browser_type="chromium",
-        headless=True,
-        browser_mode="playwright"
-    )
-    
-    # Create crawler run configuration
-    run_config = CrawlerRunConfig(
-        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-    )
-    
-    # Set up dimensions
-    browser_count = 10
-    pages_per_browser = 5
-    total_pages = browser_count * pages_per_browser
-    
-    # Create page pre-warm configuration
-    page_configs = [
-        (config, run_config, total_pages)
-    ]
-    
-    print(f"Preparing {browser_count} browsers with {pages_per_browser} pages each ({total_pages} total pages)")
-    
-    # Create browser manager with ON_DEMAND behavior as fallback
-    # No need to specify max_browsers_per_config as it will be calculated automatically
-    manager = BrowserManager(
-        browser_config=config,
-        logger=logger,
-        unavailable_behavior=UnavailableBehavior.ON_DEMAND
-    )
-    
-    try:
-        # Initialize pool with browsers and pre-warmed pages
-        print(f"Pre-warming {total_pages} pages...")
-        start_time = time.time()
-        await manager.initialize_pool(
-            browser_configs=[config],
-            browsers_per_config=browser_count,
-            page_configs=page_configs
-        )
-        warmup_time = time.time() - start_time
-        print(f"Pre-warming completed in {warmup_time:.2f} seconds")
-        
-        # Display pool status
-        status = await manager.get_pool_status()
-        print(f"Pool status after pre-warming: {status}")
-        
-        # Simulate using all pre-warmed pages simultaneously
-        print(f"\nSending {total_pages} crawl requests simultaneously...")
-        
-        async def crawl_page(index: int):
-            # url = f"https://example.com/page{index}"
-            url = SAFE_URLS[index % len(SAFE_URLS)]
-            print(f"Page {index}: Requesting page...")            
-            # Measure time to acquire page
-            page_start = time.time()
-            page, context, strategy = await manager.get_page(run_config, config)
-            page_acquisition_time = time.time() - page_start
-            
-            try:
-                # Navigate to the URL
-                nav_start = time.time()
-                await page.goto(url, timeout=5000)
-                navigation_time = time.time() - nav_start
-                
-                # Get the page title
-                title = await page.title()
-                
-                return {
-                    "index": index,
-                    "url": url,
-                    "title": title,
-                    "page_acquisition_time": page_acquisition_time,
-                    "navigation_time": navigation_time
-                }
-            except playwright._impl._errors.TimeoutError as e:
-                # print(f"Page {index}: Navigation timed out - {e}")
-                return {
-                    "index": index,
-                    "url": url,
-                    "title": "Navigation timed out",
-                    "page_acquisition_time": page_acquisition_time,
-                    "navigation_time": 0
-                }
-            finally:
-                # Release the page
-                await manager.release_page(page, strategy, config)
-        
-        # Create and execute all tasks simultaneously
-        start_time = time.time()
-
-        # Non-parallel way
-        # for i in range(total_pages):
-        #     await crawl_page(i+1)
-
-        tasks = [crawl_page(i+1) for i in range(total_pages)]
-        results = await asyncio.gather(*tasks)
-        total_time = time.time() - start_time
-        
-        # # Print all titles
-        # for result in results:
-        #     print(f"Page {result['index']} ({result['url']}): Title: {result['title']}")
-        #     print(f"  Page acquisition time: {result['page_acquisition_time']:.4f}s")
-        #     print(f"  Navigation time: {result['navigation_time']:.4f}s")
-        #     print(f"  Total time: {result['page_acquisition_time'] + result['navigation_time']:.4f}s")
-        #     print("-" * 40)
-        
-        # Report results
-        print(f"\nAll {total_pages} crawls completed in {total_time:.2f} seconds")
-        
-        # Calculate statistics
-        acquisition_times = [r["page_acquisition_time"] for r in results]
-        navigation_times = [r["navigation_time"] for r in results]
-        
-        avg_acquisition = sum(acquisition_times) / len(acquisition_times)
-        max_acquisition = max(acquisition_times)
-        min_acquisition = min(acquisition_times)
-        
-        avg_navigation = sum(navigation_times) / len(navigation_times)
-        max_navigation = max(navigation_times)
-        min_navigation = min(navigation_times)
-        
-        print("\nPage acquisition times:")
-        print(f"  Average: {avg_acquisition:.4f}s")
-        print(f"  Min: {min_acquisition:.4f}s")
-        print(f"  Max: {max_acquisition:.4f}s")
-        
-        print("\nPage navigation times:")
-        print(f"  Average: {avg_navigation:.4f}s")
-        print(f"  Min: {min_navigation:.4f}s")
-        print(f"  Max: {max_navigation:.4f}s")
-        
-        # Display final pool status
-        status = await manager.get_pool_status()
-        print(f"\nFinal pool status: {status}")
-        
-    finally:
-        # Clean up
-        print("\nClosing browser manager...")
-        await manager.close()
-        print("Browser manager closed")
-
-
-async def main():
-    """Run all demos."""
-    # await basic_pooling_demo()
-    # await prewarm_pages_demo()
-    # await prewarm_on_demand_demo()
-    await high_volume_demo()
-    # Additional demo functions can be added here
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

tests/browser/test_builtin_browser.py

@@ -56,13 +56,13 @@ async def test_builtin_browser_creation():
 
     # Step 2: Check if we have a BuiltinBrowserStrategy
     print(f"\n{INFO}2. Checking if we have a BuiltinBrowserStrategy{RESET}")
-    if isinstance(manager.strategy, BuiltinBrowserStrategy):
+    if isinstance(manager._strategy, BuiltinBrowserStrategy):
         print(
-            f"{SUCCESS}Correct strategy type: {manager.strategy.__class__.__name__}{RESET}"
+            f"{SUCCESS}Correct strategy type: {manager._strategy.__class__.__name__}{RESET}"
         )
     else:
         print(
-            f"{ERROR}Wrong strategy type: {manager.strategy.__class__.__name__}{RESET}"
+            f"{ERROR}Wrong strategy type: {manager._strategy.__class__.__name__}{RESET}"
         )
         return None
 
@@ -77,7 +77,7 @@ async def test_builtin_browser_creation():
 
     # Step 4: Get browser info from the strategy
     print(f"\n{INFO}4. Getting browser information{RESET}")
-    browser_info = manager.strategy.get_browser_info()
+    browser_info = manager._strategy.get_builtin_browser_info()
     if browser_info:
         print(f"{SUCCESS}Browser info retrieved:{RESET}")
         for key, value in browser_info.items():
@@ -149,7 +149,7 @@ async def test_browser_status_management(manager: BrowserManager):
     # Step 1: Get browser status
     print(f"\n{INFO}1. Getting browser status{RESET}")
     try:
-        status = await manager.strategy.get_builtin_browser_status()
+        status = await manager._strategy.get_builtin_browser_status()
         print(f"{SUCCESS}Browser status:{RESET}")
         print(f"  Running: {status['running']}")
         print(f"  CDP URL: {status['cdp_url']}")
@@ -160,7 +160,7 @@ async def test_browser_status_management(manager: BrowserManager):
     # Step 2: Test killing the browser
     print(f"\n{INFO}2. Testing killing the browser{RESET}")
     try:
-        result = await manager.strategy.kill_builtin_browser()
+        result = await manager._strategy.kill_builtin_browser()
         if result:
             print(f"{SUCCESS}Browser killed successfully{RESET}")
         else:
@@ -172,7 +172,7 @@ async def test_browser_status_management(manager: BrowserManager):
     # Step 3: Check status after kill
     print(f"\n{INFO}3. Checking status after kill{RESET}")
     try:
-        status = await manager.strategy.get_builtin_browser_status()
+        status = await manager._strategy.get_builtin_browser_status()
         if not status["running"]:
             print(f"{SUCCESS}Browser is correctly reported as not running{RESET}")
         else:
@@ -184,7 +184,7 @@ async def test_browser_status_management(manager: BrowserManager):
     # Step 4: Launch a new browser
     print(f"\n{INFO}4. Launching a new browser{RESET}")
     try:
-        cdp_url = await manager.strategy.launch_builtin_browser(
+        cdp_url = await manager._strategy.launch_builtin_browser(
             browser_type="chromium", headless=True
         )
         if cdp_url:
@@ -205,7 +205,7 @@ async def test_multiple_managers():
 
     # Step 1: Create first manager
     print(f"\n{INFO}1. Creating first browser manager{RESET}")
-    browser_config1 = BrowserConfig(browser_mode="builtin", headless=True)
+    browser_config1 = (BrowserConfig(browser_mode="builtin", headless=True),)
     manager1 = BrowserManager(browser_config=browser_config1, logger=logger)
 
     # Step 2: Create second manager
@@ -223,8 +223,8 @@ async def test_multiple_managers():
         print(f"{SUCCESS}Second manager started{RESET}")
 
         # Check if they got the same CDP URL
-        cdp_url1 = manager1.strategy.config.cdp_url
-        cdp_url2 = manager2.strategy.config.cdp_url
+        cdp_url1 = manager1._strategy.config.cdp_url
+        cdp_url2 = manager2._strategy.config.cdp_url
 
         if cdp_url1 == cdp_url2:
             print(
@@ -316,7 +316,7 @@ async def test_edge_cases():
 
         # Kill the browser directly
         print(f"{INFO}Killing the browser...{RESET}")
-        await manager.strategy.kill_builtin_browser()
+        await manager._strategy.kill_builtin_browser()
         print(f"{SUCCESS}Browser killed{RESET}")
 
         # Try to get a page (should fail or launch a new browser)
@@ -350,7 +350,7 @@ async def cleanup_browsers():
 
     try:
         # No need to start, just access the strategy directly
-        strategy = manager.strategy
+        strategy = manager._strategy
         if isinstance(strategy, BuiltinBrowserStrategy):
             result = await strategy.kill_builtin_browser()
             if result:
@@ -420,7 +420,7 @@ async def test_performance_scaling():
             user_data_dir=os.path.join(temp_dir, f"browser_profile_{i}"),
         )
         manager = BrowserManager(browser_config=browser_config, logger=logger)
-        manager.strategy.shutting_down = True
+        manager._strategy.shutting_down = True
         manager_configs.append((manager, i, port))
 
     # Define async function to start a single manager
@@ -614,7 +614,7 @@ async def test_performance_scaling_lab( num_browsers: int = 10, pages_per_browse
             user_data_dir=os.path.join(temp_dir, f"browser_profile_{i}"),
         )
         manager = BrowserManager(browser_config=browser_config, logger=logger)
-        manager.strategy.shutting_down = True
+        manager._strategy.shutting_down = True
         manager_configs.append((manager, i, port))
 
     # Define async function to start a single manager
@@ -781,16 +781,15 @@ async def main():
         # await manager.close()
 
         # Run multiple managers test
-        await test_multiple_managers()
+        # await test_multiple_managers()
 
         # Run performance scaling test
         await test_performance_scaling()
-
         # Run cleanup test
-        await cleanup_browsers()
+        # await cleanup_browsers()
 
         # Run edge cases test
-        await test_edge_cases()
+        # await test_edge_cases()
 
         print(f"\n{SUCCESS}All tests completed!{RESET}")
 

tests/browser/test_cdp_strategy.py

@@ -25,7 +25,6 @@ async def test_cdp_launch_connect():
     
     browser_config = BrowserConfig(
         use_managed_browser=True,
-        browser_mode="cdp",
         headless=True
     )
     
@@ -71,8 +70,8 @@ async def test_cdp_with_user_data_dir():
     logger.info(f"Created temporary user data directory: {user_data_dir}", tag="TEST")
     
     browser_config = BrowserConfig(
+        use_managed_browser=True,
         headless=True,
-        browser_mode="cdp",
         user_data_dir=user_data_dir
     )
     
@@ -211,7 +210,7 @@ async def run_tests():
     results = []
     
     # results.append(await test_cdp_launch_connect())
-    results.append(await test_cdp_with_user_data_dir())
+    # results.append(await test_cdp_with_user_data_dir())
     results.append(await test_cdp_session_management())
     
     # Print summary

tests/browser/test_playwright_strategy.py

@@ -6,7 +6,6 @@ and serve as functional tests.
 
 import asyncio
 import os
-import re
 import sys
 
 # Add the project root to Python path if running directly
@@ -20,53 +19,6 @@ from crawl4ai.async_logger import AsyncLogger
 # Create a logger for clear terminal output
 logger = AsyncLogger(verbose=True, log_file=None)
 
-
-
-async def test_start_close():
-    # Create browser config for standard Playwright
-    browser_config = BrowserConfig(
-        headless=True,
-        viewport_width=1280,
-        viewport_height=800
-    )
-    
-    # Create browser manager with the config
-    manager = BrowserManager(browser_config=browser_config, logger=logger)
-    
-    try:
-        for _ in range(4):
-            # Start the browser
-            await manager.start()
-            logger.info("Browser started successfully", tag="TEST")
-
-            # Get a page
-            page, context = await manager.get_page(CrawlerRunConfig())
-            logger.info("Got page successfully", tag="TEST")
-            
-            # Navigate to a website
-            await page.goto("https://example.com")
-            logger.info("Navigated to example.com", tag="TEST")
-            
-            # Get page title
-            title = await page.title()
-            logger.info(f"Page title: {title}", tag="TEST")
-            
-            # Clean up
-            await manager.close()
-            logger.info("Browser closed successfully", tag="TEST")
-   
-            await asyncio.sleep(1)  # Wait for a moment before restarting
-
-    except Exception as e:
-        logger.error(f"Test failed: {str(e)}", tag="TEST")
-        # Ensure cleanup
-        try:
-            await manager.close()
-        except:
-            pass
-        return False
-    return True
-
 async def test_playwright_basic():
     """Test basic Playwright browser functionality."""
     logger.info("Testing standard Playwright browser", tag="TEST")
@@ -296,10 +248,9 @@ async def run_tests():
     """Run all tests sequentially."""
     results = []
     
-    # results.append(await test_start_close())
-    # results.append(await test_playwright_basic())
-    # results.append(await test_playwright_text_mode())
-    # results.append(await test_playwright_context_reuse())
+    results.append(await test_playwright_basic())
+    results.append(await test_playwright_text_mode())
+    results.append(await test_playwright_context_reuse())
     results.append(await test_playwright_session_management())
     
     # Print summary

tests/docker/test_server_requests.py

@@ -1,655 +0,0 @@
-import pytest
-import pytest_asyncio
-import httpx
-import json
-import asyncio
-import os
-from typing import List, Dict, Any, AsyncGenerator
-
-from dotenv import load_dotenv
-load_dotenv()
-
-
-# Optional: Import crawl4ai classes directly for reference/easier payload creation aid
-# You don't strictly NEED these imports for the tests to run against the server,
-# but they help in understanding the structure you are mimicking in JSON.
-from crawl4ai import (
-    BrowserConfig,
-    CrawlerRunConfig,
-    CacheMode,
-    DefaultMarkdownGenerator,
-    PruningContentFilter,
-    BM25ContentFilter,
-    BFSDeepCrawlStrategy,
-    FilterChain,
-    ContentTypeFilter,
-    DomainFilter,
-    CompositeScorer,
-    KeywordRelevanceScorer,
-    PathDepthScorer,
-    JsonCssExtractionStrategy,
-    LLMExtractionStrategy,
-    LLMConfig
-)
-
-# --- Test Configuration ---
-# BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:8020") # Make base URL configurable
-BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:11235") # Make base URL configurable
-# Use a known simple HTML page for basic tests
-SIMPLE_HTML_URL = "https://httpbin.org/html"
-# Use a site suitable for scraping tests
-SCRAPE_TARGET_URL = "http://books.toscrape.com/"
-# Use a site with internal links for deep crawl tests
-DEEP_CRAWL_URL = "https://python.org"
-
-# --- Pytest Fixtures ---
-
-# Use the built-in event_loop fixture from pytest_asyncio
-# The custom implementation was causing issues with closing the loop
-
-@pytest_asyncio.fixture(scope="function")  # Changed to function scope to avoid event loop issues
-async def async_client() -> AsyncGenerator[httpx.AsyncClient, None]:
-    """Provides an async HTTP client"""
-    client = httpx.AsyncClient(base_url=BASE_URL, timeout=120.0)
-    yield client
-    await client.aclose()
-
-# --- Helper Functions ---
-
-async def check_server_health(client: httpx.AsyncClient):
-    """Check if the server is healthy before running tests."""
-    try:
-        response = await client.get("/health")
-        response.raise_for_status()
-        print(f"\nServer healthy: {response.json()}")
-        return True
-    except (httpx.RequestError, httpx.HTTPStatusError) as e:
-        pytest.fail(f"Server health check failed: {e}. Is the server running at {BASE_URL}?", pytrace=False)
-
-async def assert_crawl_result_structure(result: Dict[str, Any]):
-    """Asserts the basic structure of a single crawl result."""
-    assert isinstance(result, dict)
-    assert "url" in result
-    assert "success" in result
-    assert "html" in result
-    # Add more common checks if needed
-
-async def process_streaming_response(response: httpx.Response) -> List[Dict[str, Any]]:
-    """Processes an NDJSON streaming response."""
-    results = []
-    completed = False
-    async for line in response.aiter_lines():
-        if line:
-            try:
-                data = json.loads(line)
-                if data.get("status") == "completed":
-                    completed = True
-                    break # Stop processing after completion marker
-                else:
-                    results.append(data)
-            except json.JSONDecodeError:
-                pytest.fail(f"Failed to decode JSON line: {line}")
-    assert completed, "Streaming response did not end with a completion marker."
-    return results
-
-
-# --- Test Class ---
-
-@pytest.mark.asyncio
-class TestCrawlEndpoints:
-
-    @pytest_asyncio.fixture(autouse=True)
-    async def check_health_before_tests(self, async_client: httpx.AsyncClient):
-        """Fixture to ensure server is healthy before each test in the class."""
-        await check_server_health(async_client)
-
-    # 1. Simple Requests (Primitives)
-    async def test_simple_crawl_single_url(self, async_client: httpx.AsyncClient):
-        """Test /crawl with a single URL and simple config values."""
-        payload = {
-            "urls": [SIMPLE_HTML_URL],
-            "browser_config": {
-                "type": "BrowserConfig",
-                "params": {
-                    "headless": True,
-                }
-            },
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {
-                    "stream": False, # Explicitly false for /crawl
-                    "screenshot": False,
-                    "cache_mode": CacheMode.BYPASS.value # Use enum value
-                }
-            }
-        }
-        try:
-            response = await async_client.post("/crawl", json=payload)
-            print(f"Response status: {response.status_code}")
-            response.raise_for_status()
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            print(f"Server error: {e}")
-            print(f"Response content: {e.response.text}")
-            raise
-
-        assert data["success"] is True
-        assert isinstance(data["results"], list)
-        assert len(data["results"]) == 1
-        result = data["results"][0]
-        await assert_crawl_result_structure(result)
-        assert result["success"] is True
-        assert result["url"] == SIMPLE_HTML_URL
-        assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
-        # We don't specify a markdown generator in this test, so don't make assumptions about markdown field
-        # It might be null, missing, or populated depending on the server's default behavior
-
-    async def test_simple_crawl_single_url_streaming(self, async_client: httpx.AsyncClient):
-        """Test /crawl/stream with a single URL and simple config values."""
-        payload = {
-            "urls": [SIMPLE_HTML_URL],
-            "browser_config": {
-                "type": "BrowserConfig",
-                "params": {
-                    "headless": True,
-                }
-            },
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {
-                    "stream": True, # Must be true for /crawl/stream
-                    "screenshot": False,
-                    "cache_mode": CacheMode.BYPASS.value
-                }
-            }
-        }
-        async with async_client.stream("POST", "/crawl/stream", json=payload) as response:
-            response.raise_for_status()
-            results = await process_streaming_response(response)
-
-        assert len(results) == 1
-        result = results[0]
-        await assert_crawl_result_structure(result)
-        assert result["success"] is True
-        assert result["url"] == SIMPLE_HTML_URL
-        assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
-
-
-    # 2. Multi-URL and Dispatcher
-    async def test_multi_url_crawl(self, async_client: httpx.AsyncClient):
-        """Test /crawl with multiple URLs, implicitly testing dispatcher."""
-        urls = [SIMPLE_HTML_URL, "https://httpbin.org/links/10/0"]
-        payload = {
-            "urls": urls,
-            "browser_config": {
-                "type": "BrowserConfig",
-                "params": {"headless": True}
-            },
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {"stream": False, "cache_mode": CacheMode.BYPASS.value}
-            }
-        }
-        try:
-            print(f"Sending deep crawl request to server...")
-            response = await async_client.post("/crawl", json=payload)
-            print(f"Response status: {response.status_code}")
-            
-            if response.status_code >= 400:
-                error_detail = response.json().get('detail', 'No detail provided')
-                print(f"Error detail: {error_detail}")
-                print(f"Full response: {response.text}")
-            
-            response.raise_for_status()
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            print(f"Server error status: {e.response.status_code}")
-            print(f"Server error response: {e.response.text}")
-            try:
-                error_json = e.response.json()
-                print(f"Parsed error: {error_json}")
-            except:
-                print("Could not parse error response as JSON")
-            raise
-
-        assert data["success"] is True
-        assert isinstance(data["results"], list)
-        assert len(data["results"]) == len(urls)
-        for result in data["results"]:
-            await assert_crawl_result_structure(result)
-            assert result["success"] is True
-            assert result["url"] in urls
-
-    async def test_multi_url_crawl_streaming(self, async_client: httpx.AsyncClient):
-        """Test /crawl/stream with multiple URLs."""
-        urls = [SIMPLE_HTML_URL, "https://httpbin.org/links/10/0"]
-        payload = {
-            "urls": urls,
-            "browser_config": {
-                "type": "BrowserConfig",
-                "params": {"headless": True}
-            },
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {"stream": True, "cache_mode": CacheMode.BYPASS.value}
-            }
-        }
-        async with async_client.stream("POST", "/crawl/stream", json=payload) as response:
-            response.raise_for_status()
-            results = await process_streaming_response(response)
-
-        assert len(results) == len(urls)
-        processed_urls = set()
-        for result in results:
-            await assert_crawl_result_structure(result)
-            assert result["success"] is True
-            assert result["url"] in urls
-            processed_urls.add(result["url"])
-        assert processed_urls == set(urls) # Ensure all URLs were processed
-
-
-    # 3. Class Values and Nested Classes (Markdown Generator)
-    async def test_crawl_with_markdown_pruning_filter(self, async_client: httpx.AsyncClient):
-        """Test /crawl with MarkdownGenerator using PruningContentFilter."""
-        payload = {
-            "urls": [SIMPLE_HTML_URL],
-            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {
-                    "cache_mode": CacheMode.ENABLED.value, # Test different cache mode
-                    "markdown_generator": {
-                        "type": "DefaultMarkdownGenerator",
-                        "params": {
-                            "content_filter": {
-                                "type": "PruningContentFilter",
-                                "params": {
-                                    "threshold": 0.5, # Example param
-                                    "threshold_type": "relative"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        try:
-            print(f"Sending deep crawl request to server...")
-            response = await async_client.post("/crawl", json=payload)
-            print(f"Response status: {response.status_code}")
-            
-            if response.status_code >= 400:
-                error_detail = response.json().get('detail', 'No detail provided')
-                print(f"Error detail: {error_detail}")
-                print(f"Full response: {response.text}")
-            
-            response.raise_for_status()
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            print(f"Server error status: {e.response.status_code}")
-            print(f"Server error response: {e.response.text}")
-            try:
-                error_json = e.response.json()
-                print(f"Parsed error: {error_json}")
-            except:
-                print("Could not parse error response as JSON")
-            raise
-
-        assert data["success"] is True
-        assert len(data["results"]) == 1
-        result = data["results"][0]
-        await assert_crawl_result_structure(result)
-        assert result["success"] is True
-        assert "markdown" in result
-        assert isinstance(result["markdown"], dict)
-        assert "raw_markdown" in result["markdown"]
-        assert "fit_markdown" in result["markdown"] # Pruning creates fit_markdown
-        assert "Moby-Dick" in result["markdown"]["raw_markdown"]
-        # Fit markdown content might be different/shorter due to pruning
-        assert len(result["markdown"]["fit_markdown"]) <= len(result["markdown"]["raw_markdown"])
-
-    async def test_crawl_with_markdown_bm25_filter(self, async_client: httpx.AsyncClient):
-        """Test /crawl with MarkdownGenerator using BM25ContentFilter."""
-        payload = {
-            "urls": [SIMPLE_HTML_URL],
-            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {
-                    "markdown_generator": {
-                        "type": "DefaultMarkdownGenerator",
-                        "params": {
-                            "content_filter": {
-                                "type": "BM25ContentFilter",
-                                "params": {
-                                    "user_query": "Herman Melville", # Query for BM25
-                                    "bm25_threshold": 0.1, # Lower threshold to increase matches
-                                    "language": "english"  # Valid parameters
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        try:
-            print(f"Payload for BM25 test: {json.dumps(payload)}")
-            response = await async_client.post("/crawl", json=payload)
-            print(f"Response status: {response.status_code}")
-            
-            if response.status_code >= 400:
-                error_detail = response.json().get('detail', 'No detail provided')
-                print(f"Error detail: {error_detail}")
-                print(f"Full response: {response.text}")
-            
-            response.raise_for_status()
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            print(f"Server error status: {e.response.status_code}")
-            print(f"Server error response: {e.response.text}")
-            try:
-                error_json = e.response.json()
-                print(f"Parsed error: {error_json}")
-            except:
-                print("Could not parse error response as JSON")
-            raise
-
-        assert data["success"] is True
-        assert len(data["results"]) == 1
-        result = data["results"][0]
-        await assert_crawl_result_structure(result)
-        assert result["success"] is True
-        assert "markdown" in result
-        assert isinstance(result["markdown"], dict)
-        assert "raw_markdown" in result["markdown"]
-        assert "fit_markdown" in result["markdown"] # BM25 creates fit_markdown
-        
-        # Print values for debug
-        print(f"Raw markdown length: {len(result['markdown']['raw_markdown'])}")
-        print(f"Fit markdown length: {len(result['markdown']['fit_markdown'])}")
-        
-        # Either fit_markdown has content (possibly including our query terms)
-        # or it might be empty if no good BM25 matches were found
-        # Don't assert specific content since it can be environment-dependent
-
-
-    # 4. Deep Crawling
-    async def test_deep_crawl(self, async_client: httpx.AsyncClient):
-        """Test /crawl with a deep crawl strategy."""
-        payload = {
-            "urls": [DEEP_CRAWL_URL], # Start URL
-            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {
-                    "stream": False,
-                    "cache_mode": CacheMode.BYPASS.value,
-                    "deep_crawl_strategy": {
-                        "type": "BFSDeepCrawlStrategy",
-                        "params": {
-                            "max_depth": 1, # Limit depth for testing speed
-                            "max_pages": 5, # Limit pages to crawl
-                            "filter_chain": {
-                                "type": "FilterChain",
-                                "params": {
-                                    "filters": [
-                                        {
-                                            "type": "ContentTypeFilter",
-                                            "params": {"allowed_types": ["text/html"]}
-                                        },
-                                        {
-                                            "type": "DomainFilter",
-                                            "params": {"allowed_domains": ["python.org", "docs.python.org"]} # Include important subdomains
-                                        }
-                                    ]
-                                }
-                            },
-                            "url_scorer": {
-                                "type": "CompositeScorer",
-                                "params": {
-                                    "scorers": [
-                                        {
-                                            "type": "KeywordRelevanceScorer",
-                                            "params": {"keywords": ["documentation", "tutorial"]}
-                                        },
-                                        {
-                                            "type": "PathDepthScorer",
-                                            "params": {"weight": 0.5, "optimal_depth": 2}
-                                        }
-                                    ]
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        try:
-            print(f"Sending deep crawl request to server...")
-            response = await async_client.post("/crawl", json=payload)
-            print(f"Response status: {response.status_code}")
-            
-            if response.status_code >= 400:
-                error_detail = response.json().get('detail', 'No detail provided')
-                print(f"Error detail: {error_detail}")
-                print(f"Full response: {response.text}")
-            
-            response.raise_for_status()
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            print(f"Server error status: {e.response.status_code}")
-            print(f"Server error response: {e.response.text}")
-            try:
-                error_json = e.response.json()
-                print(f"Parsed error: {error_json}")
-            except:
-                print("Could not parse error response as JSON")
-            raise
-
-        assert data["success"] is True
-        assert isinstance(data["results"], list)
-        # Expect more than 1 result due to deep crawl (start URL + crawled links)
-        assert len(data["results"]) > 1
-        assert len(data["results"]) <= 6 # Start URL + max_links=5
-
-        start_url_found = False
-        crawled_urls_found = False
-        for result in data["results"]:
-            await assert_crawl_result_structure(result)
-            assert result["success"] is True
-            
-            # Print URL for debugging
-            print(f"Crawled URL: {result['url']}")
-            
-            # Allow URLs that contain python.org (including subdomains like docs.python.org)
-            assert "python.org" in result["url"]
-            if result["url"] == DEEP_CRAWL_URL:
-                start_url_found = True
-            else:
-                crawled_urls_found = True
-
-        assert start_url_found
-        assert crawled_urls_found
-
-
-    # 5. Extraction without LLM (JSON/CSS)
-    async def test_json_css_extraction(self, async_client: httpx.AsyncClient):
-        """Test /crawl with JsonCssExtractionStrategy."""
-        payload = {
-            "urls": [SCRAPE_TARGET_URL],
-            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {
-                    "cache_mode": CacheMode.BYPASS.value,
-                    "extraction_strategy": {
-                        "type": "JsonCssExtractionStrategy",
-                        "params": {
-                            "schema": { 
-                                "type": "dict", # IMPORTANT: Wrap schema dict with type/value structure
-                                "value": {
-                                    "name": "BookList",
-                                    "baseSelector": "ol.row li.col-xs-6", # Select each book item
-                                    "fields": [
-                                        {"name": "title", "selector": "article.product_pod h3 a", "type": "attribute", "attribute": "title"},
-                                        {"name": "price", "selector": "article.product_pod .price_color", "type": "text"},
-                                        {"name": "rating", "selector": "article.product_pod p.star-rating", "type": "attribute", "attribute": "class"}
-                                    ]
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        try:
-            print(f"Sending deep crawl request to server...")
-            response = await async_client.post("/crawl", json=payload)
-            print(f"Response status: {response.status_code}")
-            
-            if response.status_code >= 400:
-                error_detail = response.json().get('detail', 'No detail provided')
-                print(f"Error detail: {error_detail}")
-                print(f"Full response: {response.text}")
-            
-            response.raise_for_status()
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            print(f"Server error status: {e.response.status_code}")
-            print(f"Server error response: {e.response.text}")
-            try:
-                error_json = e.response.json()
-                print(f"Parsed error: {error_json}")
-            except:
-                print("Could not parse error response as JSON")
-            raise
-
-        assert data["success"] is True
-        assert len(data["results"]) == 1
-        result = data["results"][0]
-        await assert_crawl_result_structure(result)
-        assert result["success"] is True
-        assert "extracted_content" in result
-        assert result["extracted_content"] is not None
-
-        # Extracted content should be a JSON string representing a list of dicts
-        try:
-            extracted_data = json.loads(result["extracted_content"])
-            assert isinstance(extracted_data, list)
-            assert len(extracted_data) > 0 # Should find some books
-            # Check structure of the first extracted item
-            first_item = extracted_data[0]
-            assert "title" in first_item
-            assert "price" in first_item
-            assert "rating" in first_item
-            assert "star-rating" in first_item["rating"] # e.g., "star-rating Three"
-        except (json.JSONDecodeError, AssertionError) as e:
-            pytest.fail(f"Extracted content parsing or validation failed: {e}\nContent: {result['extracted_content']}")
-
-
-    # 6. Extraction with LLM
-    async def test_llm_extraction(self, async_client: httpx.AsyncClient):
-        """
-        Test /crawl with LLMExtractionStrategy.
-        NOTE: Requires the server to have appropriate LLM API keys (e.g., OPENAI_API_KEY)
-              configured via .llm.env or environment variables.
-              This test uses the default provider configured in the server's config.yml.
-        """
-        payload = {
-            "urls": [SIMPLE_HTML_URL],
-            "browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
-            "crawler_config": {
-                "type": "CrawlerRunConfig",
-                "params": {
-                    "cache_mode": CacheMode.BYPASS.value,
-                    "extraction_strategy": {
-                        "type": "LLMExtractionStrategy",
-                        "params": {
-                            "instruction": "Extract the main title and the author mentioned in the text into JSON.",
-                            # LLMConfig is implicitly defined by server's config.yml and .llm.env
-                            # If you needed to override provider/token PER REQUEST:
-                            "llm_config": {
-                               "type": "LLMConfig",
-                               "params": {
-                                  "provider": "openai/gpt-4o", # Example override
-                                  "api_token": os.getenv("OPENAI_API_KEY") # Example override
-                               }
-                            },
-                            "schema": { # Optional: Provide a schema for structured output
-                                "type": "dict", # IMPORTANT: Wrap schema dict
-                                "value": {
-                                    "title": "Book Info",
-                                    "type": "object",
-                                    "properties": {
-                                        "title": {"type": "string", "description": "The main title of the work"},
-                                        "author": {"type": "string", "description": "The author of the work"}
-                                    },
-                                     "required": ["title", "author"]
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        try:
-            response = await async_client.post("/crawl", json=payload)
-            response.raise_for_status() # Will raise if server returns 500 (e.g., bad API key)
-            data = response.json()
-        except httpx.HTTPStatusError as e:
-            # Catch potential server errors (like 500 due to missing/invalid API keys)
-            pytest.fail(f"LLM extraction request failed: {e}. Response: {e.response.text}. Check server logs and ensure API keys are correctly configured for the server.")
-        except httpx.RequestError as e:
-             pytest.fail(f"LLM extraction request failed: {e}.")
-
-        assert data["success"] is True
-        assert len(data["results"]) == 1
-        result = data["results"][0]
-        await assert_crawl_result_structure(result)
-        assert result["success"] is True
-        assert "extracted_content" in result
-        assert result["extracted_content"] is not None
-
-        # Extracted content should be JSON (because we provided a schema)
-        try:
-            extracted_data = json.loads(result["extracted_content"])
-            print(f"\nLLM Extracted Data: {extracted_data}") # Print for verification
-            
-            # Handle both dict and list formats (server returns a list)
-            if isinstance(extracted_data, list):
-                assert len(extracted_data) > 0
-                extracted_item = extracted_data[0]  # Take first item
-                assert isinstance(extracted_item, dict)
-                assert "title" in extracted_item
-                assert "author" in extracted_item
-                assert "Moby-Dick" in extracted_item.get("title", "")
-                assert "Herman Melville" in extracted_item.get("author", "")
-            else:
-                assert isinstance(extracted_data, dict)
-                assert "title" in extracted_data
-                assert "author" in extracted_data
-                assert "Moby-Dick" in extracted_data.get("title", "")
-                assert "Herman Melville" in extracted_data.get("author", "")
-        except (json.JSONDecodeError, AssertionError) as e:
-            pytest.fail(f"LLM extracted content parsing or validation failed: {e}\nContent: {result['extracted_content']}")
-        except Exception as e: # Catch any other unexpected error
-            pytest.fail(f"An unexpected error occurred during LLM result processing: {e}\nContent: {result['extracted_content']}")
-            
-if __name__ == "__main__":
-    # Define arguments for pytest programmatically
-    # -v: verbose output
-    # -s: show print statements immediately (useful for debugging)
-    # __file__: tells pytest to run tests in the current file
-    pytest_args = ["-v", "-s", __file__]
-
-    # You can add more pytest arguments here if needed, for example:
-    # '-k test_llm_extraction': Run only the LLM test function
-    # pytest_args.append("-k test_llm_extraction")
-
-    print(f"Running pytest with args: {pytest_args}")
-
-    # Execute pytest
-    exit_code = pytest.main(pytest_args)
-
-    print(f"Pytest finished with exit code: {exit_code}")

tests/general/test_mhtml.py

@@ -1,213 +0,0 @@
-# test_mhtml_capture.py
-
-import pytest
-import asyncio
-import re  # For more robust MHTML checks
-
-# Assuming these can be imported directly from the crawl4ai library
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CrawlResult
-
-# A reliable, simple static HTML page for testing
-# Using httpbin as it's designed for testing clients
-TEST_URL_SIMPLE = "https://httpbin.org/html"
-EXPECTED_CONTENT_SIMPLE = "Herman Melville - Moby-Dick"
-
-# A slightly more complex page that might involve JS (good secondary test)
-TEST_URL_JS = "https://quotes.toscrape.com/js/"
-EXPECTED_CONTENT_JS = "Quotes to Scrape" # Title of the page, which should be present in MHTML
-
-# Removed the custom event_loop fixture as pytest-asyncio provides a default one.
-
-@pytest.mark.asyncio
-async def test_mhtml_capture_when_enabled():
-    """
-    Verify that when CrawlerRunConfig has capture_mhtml=True,
-    the CrawlResult contains valid MHTML content.
-    """
-    # Create a fresh browser config and crawler instance for this test
-    browser_config = BrowserConfig(headless=True) # Use headless for testing CI/CD
-    # --- Key: Enable MHTML capture in the run config ---
-    run_config = CrawlerRunConfig(capture_mhtml=True)
-
-    # Create a fresh crawler instance
-    crawler = AsyncWebCrawler(config=browser_config)
-
-    try:
-        # Start the browser
-        await crawler.start()
-        
-        # Perform the crawl with the MHTML-enabled config
-        result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
-
-        # --- Assertions ---
-        assert result is not None, "Crawler should return a result object"
-        assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
-
-        # 1. Check if the mhtml attribute exists (will fail if CrawlResult not updated)
-        assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
-
-        # 2. Check if mhtml is populated
-        assert result.mhtml is not None, "MHTML content should be captured when enabled"
-        assert isinstance(result.mhtml, str), "MHTML content should be a string"
-        assert len(result.mhtml) > 500, "MHTML content seems too short, likely invalid" # Basic sanity check
-
-        # 3. Check for MHTML structure indicators (more robust than simple string contains)
-        # MHTML files are multipart MIME messages
-        assert re.search(r"Content-Type: multipart/related;", result.mhtml, re.IGNORECASE), \
-            "MHTML should contain 'Content-Type: multipart/related;'"
-        # Should contain a boundary definition
-        assert re.search(r"boundary=\"----MultipartBoundary", result.mhtml), \
-            "MHTML should contain a multipart boundary"
-        # Should contain the main HTML part
-        assert re.search(r"Content-Type: text/html", result.mhtml, re.IGNORECASE), \
-            "MHTML should contain a 'Content-Type: text/html' part"
-
-        # 4. Check if the *actual page content* is within the MHTML string
-        # This confirms the snapshot captured the rendered page
-        assert EXPECTED_CONTENT_SIMPLE in result.mhtml, \
-            f"Expected content '{EXPECTED_CONTENT_SIMPLE}' not found within the captured MHTML"
-
-        # 5. Ensure standard HTML is still present and correct
-        assert result.html is not None, "Standard HTML should still be present"
-        assert isinstance(result.html, str), "Standard HTML should be a string"
-        assert EXPECTED_CONTENT_SIMPLE in result.html, \
-            f"Expected content '{EXPECTED_CONTENT_SIMPLE}' not found within the standard HTML"
-
-    finally:
-        # Important: Ensure browser is completely closed even if assertions fail
-        await crawler.close()
-        # Help the garbage collector clean up
-        crawler = None
-
-
-@pytest.mark.asyncio
-async def test_mhtml_capture_when_disabled_explicitly():
-    """
-    Verify that when CrawlerRunConfig explicitly has capture_mhtml=False,
-    the CrawlResult.mhtml attribute is None.
-    """
-    # Create a fresh browser config and crawler instance for this test
-    browser_config = BrowserConfig(headless=True)
-    # --- Key: Explicitly disable MHTML capture ---
-    run_config = CrawlerRunConfig(capture_mhtml=False)
-
-    # Create a fresh crawler instance
-    crawler = AsyncWebCrawler(config=browser_config)
-
-    try:
-        # Start the browser
-        await crawler.start()
-        result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
-
-        assert result is not None
-        assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
-
-        # 1. Check attribute existence (important for TDD start)
-        assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
-
-        # 2. Check mhtml is None
-        assert result.mhtml is None, "MHTML content should be None when explicitly disabled"
-
-        # 3. Ensure standard HTML is still present
-        assert result.html is not None
-        assert EXPECTED_CONTENT_SIMPLE in result.html
-
-    finally:
-        # Important: Ensure browser is completely closed even if assertions fail
-        await crawler.close()
-        # Help the garbage collector clean up
-        crawler = None
-
-
-@pytest.mark.asyncio
-async def test_mhtml_capture_when_disabled_by_default():
-    """
-    Verify that if capture_mhtml is not specified (using its default),
-    the CrawlResult.mhtml attribute is None.
-    (This assumes the default value for capture_mhtml in CrawlerRunConfig is False)
-    """
-    # Create a fresh browser config and crawler instance for this test
-    browser_config = BrowserConfig(headless=True)
-    # --- Key: Use default run config ---
-    run_config = CrawlerRunConfig() # Do not specify capture_mhtml
-
-    # Create a fresh crawler instance
-    crawler = AsyncWebCrawler(config=browser_config)
-
-    try:
-        # Start the browser
-        await crawler.start()
-        result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
-
-        assert result is not None
-        assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
-
-        # 1. Check attribute existence
-        assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
-
-        # 2. Check mhtml is None (assuming default is False)
-        assert result.mhtml is None, "MHTML content should be None when using default config (assuming default=False)"
-
-        # 3. Ensure standard HTML is still present
-        assert result.html is not None
-        assert EXPECTED_CONTENT_SIMPLE in result.html
-
-    finally:
-        # Important: Ensure browser is completely closed even if assertions fail
-        await crawler.close()
-        # Help the garbage collector clean up
-        crawler = None
-
-# Optional: Add a test for a JS-heavy page if needed
-@pytest.mark.asyncio
-async def test_mhtml_capture_on_js_page_when_enabled():
-    """
-    Verify MHTML capture works on a page requiring JavaScript execution.
-    """
-    # Create a fresh browser config and crawler instance for this test
-    browser_config = BrowserConfig(headless=True)
-    run_config = CrawlerRunConfig(
-        capture_mhtml=True,
-        # Add a small wait or JS execution if needed for the JS page to fully render
-        # For quotes.toscrape.com/js/, it renders quickly, but a wait might be safer
-        # wait_for_timeout=2000 # Example: wait up to 2 seconds
-        js_code="await new Promise(r => setTimeout(r, 500));" # Small delay after potential load
-    )
-
-    # Create a fresh crawler instance
-    crawler = AsyncWebCrawler(config=browser_config)
-
-    try:
-        # Start the browser
-        await crawler.start()
-        result: CrawlResult = await crawler.arun(TEST_URL_JS, config=run_config)
-
-        assert result is not None
-        assert result.success is True, f"Crawling {TEST_URL_JS} should succeed. Error: {result.error_message}"
-        assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
-        assert result.mhtml is not None, "MHTML content should be captured on JS page when enabled"
-        assert isinstance(result.mhtml, str), "MHTML content should be a string"
-        assert len(result.mhtml) > 500, "MHTML content from JS page seems too short"
-
-        # Check for MHTML structure
-        assert re.search(r"Content-Type: multipart/related;", result.mhtml, re.IGNORECASE)
-        assert re.search(r"Content-Type: text/html", result.mhtml, re.IGNORECASE)
-
-        # Check for content rendered by JS within the MHTML
-        assert EXPECTED_CONTENT_JS in result.mhtml, \
-            f"Expected JS-rendered content '{EXPECTED_CONTENT_JS}' not found within the captured MHTML"
-
-        # Check standard HTML too
-        assert result.html is not None
-        assert EXPECTED_CONTENT_JS in result.html, \
-             f"Expected JS-rendered content '{EXPECTED_CONTENT_JS}' not found within the standard HTML"
-
-    finally:
-        # Important: Ensure browser is completely closed even if assertions fail
-        await crawler.close()
-        # Help the garbage collector clean up
-        crawler = None
-
-if __name__ == "__main__":
-    # Use pytest for async tests
-    pytest.main(["-xvs", __file__])

tests/general/test_network_console_capture.py

@@ -1,185 +0,0 @@
-from crawl4ai.async_webcrawler import AsyncWebCrawler
-from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig
-import asyncio
-import aiohttp
-from aiohttp import web
-import tempfile
-import shutil
-import os, sys, time, json
-
-
-async def start_test_server():
-    app = web.Application()
-    
-    async def basic_page(request):
-        return web.Response(text="""
-        <!DOCTYPE html>
-        <html>
-        <head>
-            <title>Network Request Test</title>
-        </head>
-        <body>
-            <h1>Test Page for Network Capture</h1>
-            <p>This page performs network requests and console logging.</p>
-            <img src="/image.png" alt="Test Image">
-            <script>
-                console.log("Basic console log");
-                console.error("Error message");
-                console.warn("Warning message");
-                
-                // Make some XHR requests
-                const xhr = new XMLHttpRequest();
-                xhr.open('GET', '/api/data', true);
-                xhr.send();
-                
-                // Make a fetch request
-                fetch('/api/json')
-                    .then(response => response.json())
-                    .catch(error => console.error('Fetch error:', error));
-                
-                // Trigger an error
-                setTimeout(() => {
-                    try {
-                        nonExistentFunction();
-                    } catch (e) {
-                        console.error("Caught error:", e);
-                    }
-                }, 100);
-            </script>
-        </body>
-        </html>
-        """, content_type="text/html")
-    
-    async def image(request):
-        # Return a small 1x1 transparent PNG
-        return web.Response(body=bytes.fromhex('89504E470D0A1A0A0000000D49484452000000010000000108060000001F15C4890000000D4944415478DA63FAFFFF3F030079DB00018D959DE70000000049454E44AE426082'), content_type="image/png")
-    
-    async def api_data(request):
-        return web.Response(text="sample data")
-    
-    async def api_json(request):
-        return web.json_response({"status": "success", "message": "JSON data"})
-    
-    # Register routes
-    app.router.add_get('/', basic_page)
-    app.router.add_get('/image.png', image)
-    app.router.add_get('/api/data', api_data)
-    app.router.add_get('/api/json', api_json)
-    
-    runner = web.AppRunner(app)
-    await runner.setup()
-    site = web.TCPSite(runner, 'localhost', 8080)
-    await site.start()
-    
-    return runner
-
-
-async def test_network_console_capture():
-    print("\n=== Testing Network and Console Capture ===\n")
-    
-    # Start test server
-    runner = await start_test_server()
-    try:
-        browser_config = BrowserConfig(headless=True)
-        
-        # Test with capture disabled (default)
-        print("\n1. Testing with capture disabled (default)...")
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            config = CrawlerRunConfig(
-                wait_until="networkidle",  # Wait for network to be idle
-            )
-            result = await crawler.arun(url="http://localhost:8080/", config=config)
-            
-            assert result.network_requests is None, "Network requests should be None when capture is disabled"
-            assert result.console_messages is None, "Console messages should be None when capture is disabled"
-            print("✓ Default config correctly returns None for network_requests and console_messages")
-        
-        # Test with network capture enabled
-        print("\n2. Testing with network capture enabled...")
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            config = CrawlerRunConfig(
-                wait_until="networkidle",  # Wait for network to be idle
-                capture_network_requests=True
-            )
-            result = await crawler.arun(url="http://localhost:8080/", config=config)
-            
-            assert result.network_requests is not None, "Network requests should be captured"
-            print(f"✓ Captured {len(result.network_requests)} network requests")
-            
-            # Check if we have both requests and responses
-            request_count = len([r for r in result.network_requests if r.get("event_type") == "request"])
-            response_count = len([r for r in result.network_requests if r.get("event_type") == "response"])
-            print(f"  - {request_count} requests, {response_count} responses")
-            
-            # Check if we captured specific resources
-            urls = [r.get("url") for r in result.network_requests]
-            has_image = any("/image.png" in url for url in urls)
-            has_api_data = any("/api/data" in url for url in urls)
-            has_api_json = any("/api/json" in url for url in urls)
-            
-            assert has_image, "Should have captured image request"
-            assert has_api_data, "Should have captured API data request"
-            assert has_api_json, "Should have captured API JSON request"
-            print("✓ Captured expected network requests (image, API endpoints)")
-        
-        # Test with console capture enabled
-        print("\n3. Testing with console capture enabled...")
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            config = CrawlerRunConfig(
-                wait_until="networkidle",  # Wait for network to be idle
-                capture_console_messages=True
-            )
-            result = await crawler.arun(url="http://localhost:8080/", config=config)
-            
-            assert result.console_messages is not None, "Console messages should be captured"
-            print(f"✓ Captured {len(result.console_messages)} console messages")
-            
-            # Check if we have different types of console messages
-            message_types = set(msg.get("type") for msg in result.console_messages if "type" in msg)
-            print(f"  - Message types: {', '.join(message_types)}")
-            
-            # Print all captured messages for debugging
-            print("  - Captured messages:")
-            for msg in result.console_messages:
-                print(f"    * Type: {msg.get('type', 'N/A')}, Text: {msg.get('text', 'N/A')}")
-            
-            # Look for specific messages
-            messages = [msg.get("text") for msg in result.console_messages if "text" in msg]
-            has_basic_log = any("Basic console log" in msg for msg in messages)
-            has_error_msg = any("Error message" in msg for msg in messages)
-            has_warning_msg = any("Warning message" in msg for msg in messages)
-            
-            assert has_basic_log, "Should have captured basic console.log message"
-            assert has_error_msg, "Should have captured console.error message"
-            assert has_warning_msg, "Should have captured console.warn message"
-            print("✓ Captured expected console messages (log, error, warning)")
-        
-        # Test with both captures enabled
-        print("\n4. Testing with both network and console capture enabled...")
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            config = CrawlerRunConfig(
-                wait_until="networkidle",  # Wait for network to be idle
-                capture_network_requests=True,
-                capture_console_messages=True
-            )
-            result = await crawler.arun(url="http://localhost:8080/", config=config)
-            
-            assert result.network_requests is not None, "Network requests should be captured"
-            assert result.console_messages is not None, "Console messages should be captured"
-            print(f"✓ Successfully captured both {len(result.network_requests)} network requests and {len(result.console_messages)} console messages")
-        
-    finally:
-        await runner.cleanup()
-        print("\nTest server shutdown")
-
-
-async def main():
-    try:
-        await test_network_console_capture()
-        print("\n✅ All tests passed successfully!")
-    except Exception as e:
-        print(f"\n❌ Test failed: {str(e)}")
-        raise
-
-if __name__ == "__main__":
-    asyncio.run(main())