Release v0.7.3: Merge release branch

- Merge release/v0.7.3 into main
- Version: 0.7.3
- Ready for tag and publication

.claude/settings.local.json

@@ -0,0 +1,28 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(cd:*)",
+      "Bash(python3:*)",
+      "Bash(python:*)",
+      "Bash(grep:*)",
+      "Bash(mkdir:*)",
+      "Bash(cp:*)",
+      "Bash(rm:*)",
+      "Bash(true)",
+      "Bash(./package-extension.sh:*)",
+      "Bash(find:*)",
+      "Bash(chmod:*)",
+      "Bash(rg:*)",
+      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -A 5 -B 5 \"Script Builder\" docs/md_v2/apps/crawl4ai-assistant/)",
+      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -A 30 \"generateCode\\(events, format\\)\" docs/md_v2/apps/crawl4ai-assistant/content/content.js)",
+      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg \"<style>\" docs/md_v2/apps/crawl4ai-assistant/index.html -A 5)",
+      "Bash(git checkout:*)",
+      "Bash(docker logs:*)",
+      "Bash(curl:*)",
+      "Bash(docker compose:*)",
+      "Bash(./test-final-integration.sh:*)",
+      "Bash(mv:*)"
+    ]
+  },
+  "enableAllProjectMcpServers": false
+}

.github/FUNDING.yml

@@ -0,0 +1,7 @@
+# These are supported funding model platforms
+
+# GitHub Sponsors
+github: unclecode
+
+# Custom links for enterprise inquiries (uncomment when ready)
+# custom: ["https://crawl4ai.com/enterprise"]

.github/workflows/main.yml

@@ -0,0 +1,46 @@
+name: Discord GitHub Notifications
+
+on:
+  issues:
+    types: [opened]
+  issue_comment:
+    types: [created]
+  pull_request:
+    types: [opened]
+  discussion:
+    types: [created]
+  watch:
+    types: [started]
+
+jobs:
+  notify-discord:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send to Google Apps Script (Stars only)
+        if: github.event_name == 'watch'
+        run: |
+          curl -fSs -X POST "${{ secrets.GOOGLE_SCRIPT_ENDPOINT }}" \
+            -H 'Content-Type: application/json' \
+            -d '{"url":"${{ github.event.sender.html_url }}"}'
+      - name: Set webhook based on event type
+        id: set-webhook
+        run: |
+          if [ "${{ github.event_name }}" == "discussion" ]; then
+            echo "webhook=${{ secrets.DISCORD_DISCUSSIONS_WEBHOOK }}" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event_name }}" == "watch" ]; then
+            echo "webhook=${{ secrets.DISCORD_STAR_GAZERS }}" >> $GITHUB_OUTPUT
+          else
+            echo "webhook=${{ secrets.DISCORD_WEBHOOK }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Discord Notification
+        uses: Ilshidur/action-discord@master
+        env:
+          DISCORD_WEBHOOK: ${{ steps.set-webhook.outputs.webhook }}
+        with:
+          args: |
+            ${{ github.event_name == 'issues' && format('📣 New issue created: **{0}** by {1} - {2}', github.event.issue.title, github.event.issue.user.login, github.event.issue.html_url) || 
+            github.event_name == 'issue_comment' && format('💬 New comment on issue **{0}** by {1} - {2}', github.event.issue.title, github.event.comment.user.login, github.event.comment.html_url) ||
+            github.event_name == 'pull_request' && format('🔄 New PR opened: **{0}** by {1} - {2}', github.event.pull_request.title, github.event.pull_request.user.login, github.event.pull_request.html_url) ||
+            github.event_name == 'watch' && format('⭐ {0} starred Crawl4AI 🥳! Check out their profile: {1}', github.event.sender.login, github.event.sender.html_url) ||
+            format('💬 New discussion started: **{0}** by {1} - {2}', github.event.discussion.title, github.event.discussion.user.login, github.event.discussion.html_url) }}

.github/workflows/release.yml

@@ -0,0 +1,142 @@
+name: Release Pipeline
+on:
+  push:
+    tags:
+      - 'v*'
+      - '!test-v*'  # Exclude test tags
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # Required for creating releases
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      
+      - name: Extract version from tag
+        id: get_version
+        run: |
+          TAG_VERSION=${GITHUB_REF#refs/tags/v}
+          echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
+          echo "Releasing version: $TAG_VERSION"
+      
+      - name: Install package dependencies
+        run: |
+          pip install -e .
+      
+      - name: Check version consistency
+        run: |
+          TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
+          PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
+          
+          echo "Tag version: $TAG_VERSION"
+          echo "Package version: $PACKAGE_VERSION"
+          
+          if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
+            echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
+            echo "Please update crawl4ai/__version__.py to match the tag version"
+            exit 1
+          fi
+          echo "✅ Version check passed: $TAG_VERSION"
+      
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+      
+      - name: Build package
+        run: python -m build
+      
+      - name: Check package
+        run: twine check dist/*
+      
+      - name: Upload to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          echo "📦 Uploading to PyPI..."
+          twine upload dist/*
+          echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/"
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      
+      - name: Extract major and minor versions
+        id: versions
+        run: |
+          VERSION=${{ steps.get_version.outputs.VERSION }}
+          MAJOR=$(echo $VERSION | cut -d. -f1)
+          MINOR=$(echo $VERSION | cut -d. -f1-2)
+          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
+          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
+      
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
+            unclecode/crawl4ai:latest
+          platforms: linux/amd64,linux/arm64
+      
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ steps.get_version.outputs.VERSION }}
+          name: Release v${{ steps.get_version.outputs.VERSION }}
+          body: |
+            ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released!
+            
+            ### 📦 Installation
+            
+            **PyPI:**
+            ```bash
+            pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}
+            ```
+            
+            **Docker:**
+            ```bash
+            docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            docker pull unclecode/crawl4ai:latest
+            ```
+            
+            ### 📝 What's Changed
+            See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
+          draft: false
+          prerelease: false
+          token: ${{ secrets.GITHUB_TOKEN }}
+      
+      - name: Summary
+        run: |
+          echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📦 PyPI Package" >> $GITHUB_STEP_SUMMARY
+          echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
+          echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
+          echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/test-release.yml.disabled

@@ -0,0 +1,116 @@
+name: Test Release Pipeline
+on:
+  push:
+    tags:
+      - 'test-v*'
+
+jobs:
+  test-release:
+    runs-on: ubuntu-latest
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      
+      - name: Extract version from tag
+        id: get_version
+        run: |
+          TAG_VERSION=${GITHUB_REF#refs/tags/test-v}
+          echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
+          echo "Testing with version: $TAG_VERSION"
+      
+      - name: Install package dependencies
+        run: |
+          pip install -e .
+      
+      - name: Check version consistency
+        run: |
+          TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
+          PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
+          
+          echo "Tag version: $TAG_VERSION"
+          echo "Package version: $PACKAGE_VERSION"
+          
+          if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
+            echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
+            echo "Please update crawl4ai/__version__.py to match the tag version"
+            exit 1
+          fi
+          echo "✅ Version check passed: $TAG_VERSION"
+      
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+      
+      - name: Build package
+        run: python -m build
+      
+      - name: Check package
+        run: twine check dist/*
+      
+      - name: Upload to Test PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }}
+        run: |
+          echo "📦 Uploading to Test PyPI..."
+          twine upload --repository testpypi dist/* || {
+            if [ $? -eq 1 ]; then
+              echo "⚠️ Upload failed - likely version already exists on Test PyPI"
+              echo "Continuing anyway for test purposes..."
+            else
+              exit 1
+            fi
+          }
+          echo "✅ Test PyPI step complete"
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      
+      - name: Build and push Docker test images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }}
+            unclecode/crawl4ai:test-latest
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      
+      - name: Summary
+        run: |
+          echo "## 🎉 Test Release Complete!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📦 Test PyPI Package" >> $GITHUB_STEP_SUMMARY
+          echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- URL: https://test.pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
+          echo "- Install: \`pip install -i https://test.pypi.org/simple/ crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🐳 Docker Test Images" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:test-latest\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🧹 Cleanup Commands" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
+          echo "# Remove test tag" >> $GITHUB_STEP_SUMMARY
+          echo "git tag -d test-v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "git push origin :test-v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "# Remove Docker test images" >> $GITHUB_STEP_SUMMARY
+          echo "docker rmi unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "docker rmi unclecode/crawl4ai:test-latest" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -1,3 +1,6 @@
+# Scripts folder (private tools)
+.scripts/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -257,4 +260,14 @@ continue_config.json
 .private/
 
 CLAUDE_MONITOR.md
-CLAUDE.md
+CLAUDE.md
+
+tests/**/test_site
+tests/**/reports
+tests/**/benchmark_reports
+
+docs/**/data
+.codecat/
+
+docs/apps/linkdin/debug*/
+docs/apps/linkdin/samples/insights/*

CHANGELOG.md

@@ -5,6 +5,177 @@ All notable changes to Crawl4AI will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.x] - 2025-06-29
+
+### Added
+- **Virtual Scroll Support**: New `VirtualScrollConfig` for handling virtualized scrolling on modern websites
+  - Automatically detects and handles three scrolling scenarios:
+    - Content unchanged (continue scrolling)
+    - Content appended (traditional infinite scroll)
+    - Content replaced (true virtual scroll - Twitter/Instagram style)
+  - Captures ALL content from pages that replace DOM elements during scroll
+  - Intelligent deduplication based on normalized text content
+  - Configurable scroll amount, count, and wait times
+  - Seamless integration with existing extraction strategies
+  - Comprehensive examples including Twitter timeline, Instagram grid, and mixed content scenarios
+
+## [Unreleased]
+
+### Added
+- **Flexible LLM Provider Configuration** (Docker): 
+  - Support for `LLM_PROVIDER` environment variable to override default provider
+  - Per-request provider override via optional `provider` parameter in API endpoints
+  - Automatic provider validation with clear error messages
+  - Updated Docker documentation and examples
+
+### Changed
+- **WebScrapingStrategy Refactoring**: Simplified content scraping architecture
+  - `WebScrapingStrategy` is now an alias for `LXMLWebScrapingStrategy` for backward compatibility
+  - Removed redundant BeautifulSoup-based implementation (~1000 lines of code)
+  - `LXMLWebScrapingStrategy` now inherits directly from `ContentScrapingStrategy`
+  - All existing code using `WebScrapingStrategy` continues to work without modification
+  - Default scraping strategy remains `LXMLWebScrapingStrategy` for optimal performance
+
+### Added
+- **AsyncUrlSeeder**: High-performance URL discovery system for intelligent crawling at scale
+  - Discover URLs from sitemaps and Common Crawl index
+  - Extract and analyze page metadata without full crawling
+  - BM25 relevance scoring for query-based URL filtering
+  - Multi-domain parallel discovery with `many_urls()` method
+  - Automatic caching with TTL for discovered URLs
+  - Rate limiting and concurrent request management
+  - Live URL validation with HEAD requests
+  - JSON-LD and Open Graph metadata extraction
+- **SeedingConfig**: Configuration class for URL seeding operations
+  - Support for multiple discovery sources (`sitemap`, `cc`, `sitemap+cc`)
+  - Pattern-based URL filtering with wildcards
+  - Configurable concurrency and rate limiting
+  - Query-based relevance scoring with BM25
+  - Score threshold filtering for quality control
+- Comprehensive documentation for URL seeding feature
+  - Detailed comparison with deep crawling approaches
+  - Complete API reference with examples
+  - Integration guide with AsyncWebCrawler
+  - Performance benchmarks and best practices
+- Example scripts demonstrating URL seeding:
+  - `url_seeder_demo.py`: Interactive Rich-based demonstration
+  - `url_seeder_quick_demo.py`: Screenshot-friendly examples
+- Test suite for URL seeding with BM25 scoring
+
+### Changed
+- Updated `__init__.py` to export AsyncUrlSeeder and SeedingConfig
+- Enhanced documentation with URL seeding integration examples
+
+### Fixed
+- Corrected examples to properly extract URLs from seeder results before passing to `arun_many()`
+- Fixed logger color compatibility issue (changed `lightblack` to `bright_black`)
+
+## [0.6.2] - 2025-05-02
+
+### Added
+- New `RegexExtractionStrategy` for fast pattern-based extraction without requiring LLM
+  - Built-in patterns for emails, URLs, phone numbers, dates, and more
+  - Support for custom regex patterns
+  - `generate_pattern` utility for LLM-assisted pattern creation (one-time use)
+- Added `fit_html` as a top-level field in `CrawlResult` for optimized HTML extraction
+- Added support for network response body capture in network request tracking
+
+### Changed
+- Updated documentation for no-LLM extraction strategies
+- Enhanced API reference to include RegexExtractionStrategy examples and usage
+- Improved HTML preprocessing with optimized performance for extraction strategies
+
+## [0.6.1] - 2025-04-24
+
+### Added
+- New dedicated `tables` field in `CrawlResult` model for better table extraction handling
+- Updated crypto_analysis_example.py to use the new tables field with backward compatibility
+
+### Changed
+- Improved playground UI in Docker deployment with better endpoint handling and UI feedback
+
+## [0.6.0] ‑ 2025‑04‑22
+
+### Added
+- Browser pooling with page pre‑warming and fine‑grained **geolocation, locale, and timezone** controls  
+- Crawler pool manager (SDK + Docker API) for smarter resource allocation  
+- Network & console log capture plus MHTML snapshot export  
+- **Table extractor**: turn HTML `<table>`s into DataFrames or CSV with one flag  
+- High‑volume stress‑test framework in `tests/memory` and API load scripts  
+- MCP protocol endpoints with socket & SSE support; playground UI scaffold  
+- Docs v2 revamp: TOC, GitHub badge, copy‑code buttons, Docker API demo  
+- “Ask AI” helper button *(work‑in‑progress, shipping soon)*  
+- New examples: geo‑location usage, network/console capture, Docker API, markdown source selection, crypto analysis  
+- Expanded automated test suites for browser, Docker, MCP and memory benchmarks  
+
+### Changed
+- Consolidated and renamed browser strategies; legacy docker strategy modules removed  
+- `ProxyConfig` moved to `async_configs`  
+- Server migrated to pool‑based crawler management  
+- FastAPI validators replace custom query validation  
+- Docker build now uses Chromium base image  
+- Large‑scale repo tidy‑up (≈36 k insertions, ≈5 k deletions)  
+
+### Fixed
+- Async crawler session leak, duplicate‑visit handling, URL normalisation  
+- Target‑element regressions in scraping strategies  
+- Logged‑URL readability, encoded‑URL decoding, middle truncation for long URLs  
+- Closed issues: #701, #733, #756, #774, #804, #822, #839, #841, #842, #843, #867, #902, #911  
+
+### Removed
+- Obsolete modules under `crawl4ai/browser/*` superseded by the new pooled browser layer  
+
+### Deprecated
+- Old markdown generator names now alias `DefaultMarkdownGenerator` and emit warnings  
+
+---
+
+#### Upgrade notes
+1. Update any direct imports from `crawl4ai/browser/*` to the new pooled browser modules  
+2. If you override `AsyncPlaywrightCrawlerStrategy.get_page`, adopt the new signature  
+3. Rebuild Docker images to pull the new Chromium layer  
+4. Switch to `DefaultMarkdownGenerator` (or silence the deprecation warning)  
+
+---
+
+`121 files changed, ≈36 223 insertions, ≈4 975 deletions` :contentReference[oaicite:0]{index=0}&#8203;:contentReference[oaicite:1]{index=1}
+
+
+### [Feature] 2025-04-21
+- Implemented MCP protocol for machine-to-machine communication
+  - Added WebSocket and SSE transport for MCP server
+  - Exposed server endpoints via MCP protocol
+  - Created tests for MCP socket and SSE communication
+- Enhanced Docker server with file handling and intelligent search
+  - Added PDF and screenshot endpoints with file saving capability
+  - Added JavaScript execution endpoint for page interaction
+  - Implemented advanced context search with BM25 and code chunking
+  - Added file path output support for generated assets
+- Improved server endpoints and API surface
+  - Added intelligent context search with query filtering
+  - Added syntax-aware code function chunking
+  - Implemented efficient HTML processing pipeline
+- Added support for controlling browser geolocation via new GeolocationConfig class
+  - Added locale and timezone configuration options to CrawlerRunConfig
+  - Added example script demonstrating geolocation and locale usage
+  - Added documentation for location-based identity features
+
+### [Refactor] 2025-04-20
+- Replaced crawler_manager.py with simpler crawler_pool.py implementation
+- Added global page semaphore for hard concurrency cap
+- Implemented browser pool with idle cleanup
+- Added playground UI for testing and stress testing
+- Updated API handlers to use pooled crawlers
+- Enhanced logging levels and symbols
+- Added memory tests and stress test utilities
+
+### [Added] 2025-04-17
+- Added content source selection feature for markdown generation
+  - New `content_source` parameter allows choosing between `cleaned_html`, `raw_html`, and `fit_html`
+  - Provides flexibility in how HTML content is processed before markdown conversion
+  - Added examples and documentation for the new feature
+  - Includes backward compatibility with default `cleaned_html` behavior
+  
 ## Version 0.5.0.post5 (2025-03-14)
 
 ### Added

Dockerfile

@@ -1,4 +1,9 @@
-FROM python:3.10-slim
+FROM python:3.12-slim-bookworm AS build
+
+# C4ai version
+ARG C4AI_VER=0.7.0-r1
+ENV C4AI_VERSION=$C4AI_VER
+LABEL c4ai.version=$C4AI_VER
 
 # Set build arguments
 ARG APP_HOME=/app
@@ -17,14 +22,14 @@ ENV PYTHONFAULTHANDLER=1 \
     REDIS_HOST=localhost \
     REDIS_PORT=6379
 
-ARG PYTHON_VERSION=3.10
+ARG PYTHON_VERSION=3.12
 ARG INSTALL_TYPE=default
 ARG ENABLE_GPU=false
 ARG TARGETARCH
 
 LABEL maintainer="unclecode"
 LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
-LABEL version="1.0"    
+LABEL version="1.0"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
@@ -38,6 +43,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libjpeg-dev \
     redis-server \
     supervisor \
+    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/*
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -62,11 +68,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libcairo2 \
     libasound2 \
     libatspi2.0-0 \
+    && apt-get clean \ 
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get dist-upgrade -y \
     && rm -rf /var/lib/apt/lists/*
 
 RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \
     apt-get update && apt-get install -y --no-install-recommends \
     nvidia-cuda-toolkit \
+    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/* ; \
 else \
     echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
@@ -76,16 +87,24 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \
     echo "🦾 Installing ARM-specific optimizations"; \
     apt-get update && apt-get install -y --no-install-recommends \
     libopenblas-dev \
+    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/*; \
 elif [ "$TARGETARCH" = "amd64" ]; then \
     echo "🖥️ Installing AMD64-specific optimizations"; \
     apt-get update && apt-get install -y --no-install-recommends \
     libomp-dev \
+    && apt-get clean \ 
     && rm -rf /var/lib/apt/lists/*; \
 else \
     echo "Skipping platform-specific optimizations (unsupported platform)"; \
 fi
 
+# Create a non-root user and group
+RUN groupadd -r appuser && useradd --no-log-init -r -g appuser appuser
+
+# Create and set permissions for appuser home directory
+RUN mkdir -p /home/appuser && chown -R appuser:appuser /home/appuser
+
 WORKDIR ${APP_HOME}
 
 RUN echo '#!/bin/bash\n\
@@ -103,6 +122,7 @@ fi' > /tmp/install.sh && chmod +x /tmp/install.sh
 
 COPY . /tmp/project/
 
+# Copy supervisor config first (might need root later, but okay for now)
 COPY deploy/docker/supervisord.conf .
 
 COPY deploy/docker/requirements.txt .
@@ -131,16 +151,34 @@ RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
     else \
         pip install "/tmp/project" ; \
     fi
-    
+
 RUN pip install --no-cache-dir --upgrade pip && \
     /tmp/install.sh && \
     python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
     python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')"
-    
-RUN playwright install --with-deps chromium
 
+RUN crawl4ai-setup
+
+RUN playwright install --with-deps
+
+RUN mkdir -p /home/appuser/.cache/ms-playwright \
+    && cp -r /root/.cache/ms-playwright/chromium-* /home/appuser/.cache/ms-playwright/ \
+    && chown -R appuser:appuser /home/appuser/.cache/ms-playwright
+
+RUN crawl4ai-doctor
+
+# Copy application code
 COPY deploy/docker/* ${APP_HOME}/
 
+# copy the playground + any future static assets
+COPY deploy/docker/static ${APP_HOME}/static
+
+# Change ownership of the application directory to the non-root user
+RUN chown -R appuser:appuser ${APP_HOME}
+
+# give permissions to redis persistence dirs if used
+RUN mkdir -p /var/lib/redis /var/log/redis && chown -R appuser:appuser /var/lib/redis /var/log/redis
+
 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
     CMD bash -c '\
     MEM=$(free -m | awk "/^Mem:/{print \$2}"); \
@@ -149,8 +187,14 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
         exit 1; \
     fi && \
     redis-cli ping > /dev/null && \
-    curl -f http://localhost:8000/health || exit 1'
+    curl -f http://localhost:11235/health || exit 1'
 
 EXPOSE 6379
-CMD ["supervisord", "-c", "supervisord.conf"]
-    
+# Switch to the non-root user before starting the application
+USER appuser
+
+# Set environment variables to ptoduction
+ENV PYTHON_ENV=production 
+
+# Start the application using supervisord
+CMD ["supervisord", "-c", "supervisord.conf"]

JOURNAL.md

@@ -0,0 +1,339 @@
+# Development Journal
+
+This journal tracks significant feature additions, bug fixes, and architectural decisions in the crawl4ai project. It serves as both documentation and a historical record of the project's evolution.
+
+## [2025-04-17] Added Content Source Selection for Markdown Generation
+
+**Feature:** Configurable content source for markdown generation
+
+**Changes Made:**
+1. Added `content_source: str = "cleaned_html"` parameter to `MarkdownGenerationStrategy` class
+2. Updated `DefaultMarkdownGenerator` to accept and pass the content source parameter
+3. Renamed the `cleaned_html` parameter to `input_html` in the `generate_markdown` method
+4. Modified `AsyncWebCrawler.aprocess_html` to select the appropriate HTML source based on the generator's config
+5. Added `preprocess_html_for_schema` import in `async_webcrawler.py`
+
+**Implementation Details:**
+- Added a new `content_source` parameter to specify which HTML input to use for markdown generation
+- Options include: "cleaned_html" (default), "raw_html", and "fit_html"
+- Used a dictionary dispatch pattern in `aprocess_html` to select the appropriate HTML source
+- Added proper error handling with fallback to cleaned_html if content source selection fails
+- Ensured backward compatibility by defaulting to "cleaned_html" option
+
+**Files Modified:**
+- `crawl4ai/markdown_generation_strategy.py`: Added content_source parameter and updated the method signature
+- `crawl4ai/async_webcrawler.py`: Added HTML source selection logic and updated imports
+
+**Examples:**
+- Created `docs/examples/content_source_example.py` demonstrating how to use the new parameter
+
+**Challenges:**
+- Maintaining backward compatibility while reorganizing the parameter flow
+- Ensuring proper error handling for all content source options
+- Making the change with minimal code modifications
+
+**Why This Feature:**
+The content source selection feature allows users to choose which HTML content to use as input for markdown generation:
+1. "cleaned_html" - Uses the post-processed HTML after scraping strategy (original behavior)
+2. "raw_html" - Uses the original raw HTML directly from the web page
+3. "fit_html" - Uses the preprocessed HTML optimized for schema extraction
+
+This feature provides greater flexibility in how users generate markdown, enabling them to:
+- Capture more detailed content from the original HTML when needed
+- Use schema-optimized HTML when working with structured data
+- Choose the approach that best suits their specific use case
+## [2025-04-17] Implemented High Volume Stress Testing Solution for SDK
+
+**Feature:** Comprehensive stress testing framework using `arun_many` and the dispatcher system to evaluate performance, concurrency handling, and identify potential issues under high-volume crawling scenarios.
+
+**Changes Made:**
+1.  Created a dedicated stress testing framework in the `benchmarking/` (or similar) directory.
+2.  Implemented local test site generation (`SiteGenerator`) with configurable heavy HTML pages.
+3.  Added basic memory usage tracking (`SimpleMemoryTracker`) using platform-specific commands (avoiding `psutil` dependency for this specific test).
+4.  Utilized `CrawlerMonitor` from `crawl4ai` for rich terminal UI and real-time monitoring of test progress and dispatcher activity.
+5.  Implemented detailed result summary saving (JSON) and memory sample logging (CSV).
+6.  Developed `run_benchmark.py` to orchestrate tests with predefined configurations.
+7.  Created `run_all.sh` as a simple wrapper for `run_benchmark.py`.
+
+**Implementation Details:**
+-   Generates a local test site with configurable pages containing heavy text and image content.
+-   Uses Python's built-in `http.server` for local serving, minimizing network variance.
+-   Leverages `crawl4ai`'s `arun_many` method for processing URLs.
+-   Utilizes `MemoryAdaptiveDispatcher` to manage concurrency via the `max_sessions` parameter (note: memory adaptation features require `psutil`, not used by `SimpleMemoryTracker`).
+-   Tracks memory usage via `SimpleMemoryTracker`, recording samples throughout test execution to a CSV file.
+-   Uses `CrawlerMonitor` (which uses the `rich` library) for clear terminal visualization and progress reporting directly from the dispatcher.
+-   Stores detailed final metrics in a JSON summary file.
+
+**Files Created/Updated:**
+-   `stress_test_sdk.py`: Main stress testing implementation using `arun_many`.
+-   `benchmark_report.py`: (Assumed) Report generator for comparing test results.
+-   `run_benchmark.py`: Test runner script with predefined configurations.
+-   `run_all.sh`: Simple bash script wrapper for `run_benchmark.py`.
+-   `USAGE.md`: Comprehensive documentation on usage and interpretation (updated).
+
+**Testing Approach:**
+-   Creates a controlled, reproducible test environment with a local HTTP server.
+-   Processes URLs using `arun_many`, allowing the dispatcher to manage concurrency up to `max_sessions`.
+-   Optionally logs per-batch summaries (when not in streaming mode) after processing chunks.
+-   Supports different test sizes via `run_benchmark.py` configurations.
+-   Records memory samples via platform commands for basic trend analysis.
+-   Includes cleanup functionality for the test environment.
+
+**Challenges:**
+-   Ensuring proper cleanup of HTTP server processes.
+-   Getting reliable memory tracking across platforms without adding heavy dependencies (`psutil`) to this specific test script.
+-   Designing `run_benchmark.py` to correctly pass arguments to `stress_test_sdk.py`.
+
+**Why This Feature:**
+The high volume stress testing solution addresses critical needs for ensuring Crawl4AI's `arun_many` reliability:
+1.  Provides a reproducible way to evaluate performance under concurrent load.
+2.  Allows testing the dispatcher's concurrency control (`max_session_permit`) and queue management.
+3.  Enables performance tuning by observing throughput (`URLs/sec`) under different `max_sessions` settings.
+4.  Creates a controlled environment for testing `arun_many` behavior.
+5.  Supports continuous integration by providing deterministic test conditions for `arun_many`.
+
+**Design Decisions:**
+-   Chose local site generation for reproducibility and isolation from network issues.
+-   Utilized the built-in `CrawlerMonitor` for real-time feedback, leveraging its `rich` integration.
+-   Implemented optional per-batch logging in `stress_test_sdk.py` (when not streaming) to provide chunk-level summaries alongside the continuous monitor.
+-   Adopted `arun_many` with a `MemoryAdaptiveDispatcher` as the core mechanism for parallel execution, reflecting the intended SDK usage.
+-   Created `run_benchmark.py` to simplify running standard test configurations.
+-   Used `SimpleMemoryTracker` to provide basic memory insights without requiring `psutil` for this particular test runner.
+
+**Future Enhancements to Consider:**
+-   Create a separate test variant that *does* use `psutil` to specifically stress the memory-adaptive features of the dispatcher.
+-   Add support for generated JavaScript content.
+-   Add support for Docker-based testing with explicit memory limits.
+-   Enhance `benchmark_report.py` to provide more sophisticated analysis of performance and memory trends from the generated JSON/CSV files.
+
+---
+
+## [2025-04-17] Refined Stress Testing System Parameters and Execution
+
+**Changes Made:**
+1.  Corrected `run_benchmark.py` and `stress_test_sdk.py` to use `--max-sessions` instead of the incorrect `--workers` parameter, accurately reflecting dispatcher configuration.
+2.  Updated `run_benchmark.py` argument handling to correctly pass all relevant custom parameters (including `--stream`, `--monitor-mode`, etc.) to `stress_test_sdk.py`.
+3.  (Assuming changes in `benchmark_report.py`) Applied dark theme to benchmark reports for better readability.
+4.  (Assuming changes in `benchmark_report.py`) Improved visualization code to eliminate matplotlib warnings.
+5.  Updated `run_benchmark.py` to provide clickable `file://` links to generated reports in the terminal output.
+6.  Updated `USAGE.md` with comprehensive parameter descriptions reflecting the final script arguments.
+7.  Updated `run_all.sh` wrapper to correctly invoke `run_benchmark.py` with flexible arguments.
+
+**Details of Changes:**
+
+1.  **Parameter Correction (`--max-sessions`)**:
+    *   Identified the fundamental misunderstanding where `--workers` was used incorrectly.
+    *   Refactored `stress_test_sdk.py` to accept `--max-sessions` and configure the `MemoryAdaptiveDispatcher`'s `max_session_permit` accordingly.
+    *   Updated `run_benchmark.py` argument parsing and command construction to use `--max-sessions`.
+    *   Updated `TEST_CONFIGS` in `run_benchmark.py` to use `max_sessions`.
+
+2.  **Argument Handling (`run_benchmark.py`)**:
+    *   Improved logic to collect all command-line arguments provided to `run_benchmark.py`.
+    *   Ensured all relevant arguments (like `--stream`, `--monitor-mode`, `--port`, `--use-rate-limiter`, etc.) are correctly forwarded when calling `stress_test_sdk.py` as a subprocess.
+
+3.  **Dark Theme & Visualization Fixes (Assumed in `benchmark_report.py`)**:
+    *   (Describes changes assumed to be made in the separate reporting script).
+
+4.  **Clickable Links (`run_benchmark.py`)**:
+    *   Added logic to find the latest HTML report and PNG chart in the `benchmark_reports` directory after `benchmark_report.py` runs.
+    *   Used `pathlib` to generate correct `file://` URLs for terminal output.
+
+5.  **Documentation Improvements (`USAGE.md`)**:
+    *   Rewrote sections to explain `arun_many`, dispatchers, and `--max-sessions`.
+    *   Updated parameter tables for all scripts (`stress_test_sdk.py`, `run_benchmark.py`).
+    *   Clarified the difference between batch and streaming modes and their effect on logging.
+    *   Updated examples to use correct arguments.
+
+**Files Modified:**
+-   `stress_test_sdk.py`: Changed `--workers` to `--max-sessions`, added new arguments, used `arun_many`.
+-   `run_benchmark.py`: Changed argument handling, updated configs, calls `stress_test_sdk.py`.
+-   `run_all.sh`: Updated to call `run_benchmark.py` correctly.
+-   `USAGE.md`: Updated documentation extensively.
+-   `benchmark_report.py`: (Assumed modifications for dark theme and viz fixes).
+
+**Testing:**
+-   Verified that `--max-sessions` correctly limits concurrency via the `CrawlerMonitor` output.
+-   Confirmed that custom arguments passed to `run_benchmark.py` are forwarded to `stress_test_sdk.py`.
+-   Validated clickable links work in supporting terminals.
+-   Ensured documentation matches the final script parameters and behavior.
+
+**Why These Changes:**
+These refinements correct the fundamental approach of the stress test to align with `crawl4ai`'s actual architecture and intended usage:
+1.  Ensures the test evaluates the correct components (`arun_many`, `MemoryAdaptiveDispatcher`).
+2.  Makes test configurations more accurate and flexible.
+3.  Improves the usability of the testing framework through better argument handling and documentation.
+
+
+**Future Enhancements to Consider:**
+- Add support for generated JavaScript content to test JS rendering performance
+- Implement more sophisticated memory analysis like generational garbage collection tracking
+- Add support for Docker-based testing with memory limits to force OOM conditions
+- Create visualization tools for analyzing memory usage patterns across test runs
+- Add benchmark comparisons between different crawler versions or configurations
+
+## [2025-04-17] Fixed Issues in Stress Testing System
+
+**Changes Made:**
+1. Fixed custom parameter handling in run_benchmark.py
+2. Applied dark theme to benchmark reports for better readability
+3. Improved visualization code to eliminate matplotlib warnings
+4. Added clickable links to generated reports in terminal output
+5. Enhanced documentation with comprehensive parameter descriptions
+
+**Details of Changes:**
+
+1. **Custom Parameter Handling Fix**
+   - Identified bug where custom URL count was being ignored in run_benchmark.py
+   - Rewrote argument handling to use a custom args dictionary
+   - Properly passed parameters to the test_simple_stress.py command
+   - Added better UI indication of custom parameters in use
+
+2. **Dark Theme Implementation**
+   - Added complete dark theme to HTML benchmark reports
+   - Applied dark styling to all visualization components
+   - Used Nord-inspired color palette for charts and graphs
+   - Improved contrast and readability for data visualization
+   - Updated text colors and backgrounds for better eye comfort
+
+3. **Matplotlib Warning Fixes**
+   - Resolved warnings related to improper use of set_xticklabels()
+   - Implemented correct x-axis positioning for bar charts
+   - Ensured proper alignment of bar labels and data points
+   - Updated plotting code to use modern matplotlib practices
+
+4. **Documentation Improvements**
+   - Created comprehensive USAGE.md with detailed instructions
+   - Added parameter documentation for all scripts
+   - Included examples for all common use cases
+   - Provided detailed explanations for interpreting results
+   - Added troubleshooting guide for common issues
+
+**Files Modified:**
+- `tests/memory/run_benchmark.py`: Fixed custom parameter handling
+- `tests/memory/benchmark_report.py`: Added dark theme and fixed visualization warnings
+- `tests/memory/run_all.sh`: Added clickable links to reports
+- `tests/memory/USAGE.md`: Created comprehensive documentation
+
+**Testing:**
+- Verified that custom URL counts are now correctly used
+- Confirmed dark theme is properly applied to all report elements
+- Checked that matplotlib warnings are no longer appearing
+- Validated clickable links to reports work in terminals that support them
+
+**Why These Changes:**
+These improvements address several usability issues with the stress testing system:
+1. Better parameter handling ensures test configurations work as expected
+2. Dark theme reduces eye strain during extended test review sessions
+3. Fixing visualization warnings improves code quality and output clarity
+4. Enhanced documentation makes the system more accessible for future use
+
+**Future Enhancements:**
+- Add additional visualization options for different types of analysis
+- Implement theme toggle to support both light and dark preferences
+- Add export options for embedding reports in other documentation
+- Create dedicated CI/CD integration templates for automated testing
+
+## [2025-04-09] Added MHTML Capture Feature
+
+**Feature:** MHTML snapshot capture of crawled pages
+
+**Changes Made:**
+1. Added `capture_mhtml: bool = False` parameter to `CrawlerRunConfig` class
+2. Added `mhtml: Optional[str] = None` field to `CrawlResult` model
+3. Added `mhtml_data: Optional[str] = None` field to `AsyncCrawlResponse` class
+4. Implemented `capture_mhtml()` method in `AsyncPlaywrightCrawlerStrategy` class to capture MHTML via CDP
+5. Modified the crawler to capture MHTML when enabled and pass it to the result
+
+**Implementation Details:**
+- MHTML capture uses Chrome DevTools Protocol (CDP) via Playwright's CDP session API
+- The implementation waits for page to fully load before capturing MHTML content
+- Enhanced waiting for JavaScript content with requestAnimationFrame for better JS content capture
+- We ensure all browser resources are properly cleaned up after capture
+
+**Files Modified:**
+- `crawl4ai/models.py`: Added the mhtml field to CrawlResult
+- `crawl4ai/async_configs.py`: Added capture_mhtml parameter to CrawlerRunConfig
+- `crawl4ai/async_crawler_strategy.py`: Implemented MHTML capture logic
+- `crawl4ai/async_webcrawler.py`: Added mapping from AsyncCrawlResponse.mhtml_data to CrawlResult.mhtml
+
+**Testing:**
+- Created comprehensive tests in `tests/20241401/test_mhtml.py` covering:
+  - Capturing MHTML when enabled
+  - Ensuring mhtml is None when disabled explicitly
+  - Ensuring mhtml is None by default
+  - Capturing MHTML on JavaScript-enabled pages
+
+**Challenges:**
+- Had to improve page loading detection to ensure JavaScript content was fully rendered
+- Tests needed to be run independently due to Playwright browser instance management
+- Modified test expected content to match actual MHTML output
+
+**Why This Feature:**
+The MHTML capture feature allows users to capture complete web pages including all resources (CSS, images, etc.) in a single file. This is valuable for:
+1. Offline viewing of captured pages
+2. Creating permanent snapshots of web content for archival
+3. Ensuring consistent content for later analysis, even if the original site changes
+
+**Future Enhancements to Consider:**
+- Add option to save MHTML to file
+- Support for filtering what resources get included in MHTML
+- Add support for specifying MHTML capture options
+
+## [2025-04-10] Added Network Request and Console Message Capturing
+
+**Feature:** Comprehensive capturing of network requests/responses and browser console messages during crawling
+
+**Changes Made:**
+1. Added `capture_network_requests: bool = False` and `capture_console_messages: bool = False` parameters to `CrawlerRunConfig` class
+2. Added `network_requests: Optional[List[Dict[str, Any]]] = None` and `console_messages: Optional[List[Dict[str, Any]]] = None` fields to both `AsyncCrawlResponse` and `CrawlResult` models
+3. Implemented event listeners in `AsyncPlaywrightCrawlerStrategy._crawl_web()` to capture browser network events and console messages
+4. Added proper event listener cleanup in the finally block to prevent resource leaks
+5. Modified the crawler flow to pass captured data from AsyncCrawlResponse to CrawlResult
+
+**Implementation Details:**
+- Network capture uses Playwright event listeners (`request`, `response`, and `requestfailed`) to record all network activity
+- Console capture uses Playwright event listeners (`console` and `pageerror`) to record console messages and errors
+- Each network event includes metadata like URL, headers, status, and timing information
+- Each console message includes type, text content, and source location when available
+- All captured events include timestamps for chronological analysis
+- Error handling ensures even failed capture attempts won't crash the main crawling process
+
+**Files Modified:**
+- `crawl4ai/models.py`: Added new fields to AsyncCrawlResponse and CrawlResult
+- `crawl4ai/async_configs.py`: Added new configuration parameters to CrawlerRunConfig
+- `crawl4ai/async_crawler_strategy.py`: Implemented capture logic using event listeners
+- `crawl4ai/async_webcrawler.py`: Added data transfer from AsyncCrawlResponse to CrawlResult
+
+**Documentation:**
+- Created detailed documentation in `docs/md_v2/advanced/network-console-capture.md`
+- Added feature to site navigation in `mkdocs.yml`
+- Updated CrawlResult documentation in `docs/md_v2/api/crawl-result.md`
+- Created comprehensive example in `docs/examples/network_console_capture_example.py`
+
+**Testing:**
+- Created `tests/general/test_network_console_capture.py` with tests for:
+  - Verifying capture is disabled by default
+  - Testing network request capturing
+  - Testing console message capturing
+  - Ensuring both capture types can be enabled simultaneously
+  - Checking correct content is captured in expected formats
+
+**Challenges:**
+- Initial implementation had synchronous/asynchronous mismatches in event handlers
+- Needed to fix type of property access vs. method calls in handlers
+- Required careful cleanup of event listeners to prevent memory leaks
+
+**Why This Feature:**
+The network and console capture feature provides deep visibility into web page activity, enabling:
+1. Debugging complex web applications by seeing all network requests and errors
+2. Security analysis to detect unexpected third-party requests and data flows
+3. Performance profiling to identify slow-loading resources
+4. API discovery in single-page applications
+5. Comprehensive analysis of web application behavior
+
+**Future Enhancements to Consider:**
+- Option to filter captured events by type, domain, or content
+- Support for capturing response bodies (with size limits)
+- Aggregate statistics calculation for performance metrics
+- Integration with visualization tools for network waterfall analysis
+- Exporting captures in HAR format for use with external tools

PROGRESSIVE_CRAWLING.md

@@ -0,0 +1,320 @@
+# Progressive Web Crawling with Adaptive Information Foraging
+
+## Abstract
+
+This paper presents a novel approach to web crawling that adaptively determines when sufficient information has been gathered to answer a given query. Unlike traditional exhaustive crawling methods, our Progressive Information Sufficiency (PIS) framework uses statistical measures to balance information completeness against crawling efficiency. We introduce a multi-strategy architecture supporting pure statistical, embedding-enhanced, and LLM-assisted approaches, with theoretical guarantees on convergence and practical evaluation methods using synthetic datasets.
+
+## 1. Introduction
+
+Traditional web crawling approaches follow predetermined patterns (breadth-first, depth-first) without consideration for information sufficiency. This work addresses the fundamental question: *"When do we have enough information to answer a query and similar queries in its domain?"*
+
+We formalize this as an optimal stopping problem in information foraging, introducing metrics for coverage, consistency, and saturation that enable crawlers to make intelligent decisions about when to stop crawling and which links to follow.
+
+## 2. Problem Formulation
+
+### 2.1 Definitions
+
+Let:
+- **K** = {d₁, d₂, ..., dₙ} be the current knowledge base (crawled documents)
+- **Q** be the user query
+- **L** = {l₁, l₂, ..., lₘ} be available links with preview metadata
+- **θ** be the confidence threshold for information sufficiency
+
+### 2.2 Objectives
+
+1. **Minimize** |K| (number of crawled pages)
+2. **Maximize** P(answers(Q) | K) (probability of answering Q given K)
+3. **Ensure** coverage of Q's domain (similar queries)
+
+## 3. Mathematical Framework
+
+### 3.1 Information Sufficiency Metric
+
+We define Information Sufficiency as:
+
+```
+IS(K, Q) = min(Coverage(K, Q), Consistency(K, Q), 1 - Redundancy(K)) × DomainCoverage(K, Q)
+```
+
+### 3.2 Coverage Score
+
+Coverage measures how well current knowledge covers query terms and related concepts:
+
+```
+Coverage(K, Q) = Σ(t ∈ Q) log(df(t, K) + 1) × idf(t) / |Q|
+```
+
+Where:
+- df(t, K) = document frequency of term t in knowledge base K
+- idf(t) = inverse document frequency weight
+
+### 3.3 Consistency Score
+
+Consistency measures information coherence across documents:
+
+```
+Consistency(K, Q) = 1 - Var(answers from random subsets of K)
+```
+
+This captures the principle that sufficient knowledge should provide stable answers regardless of document subset.
+
+### 3.4 Saturation Score
+
+Saturation detects diminishing returns:
+
+```
+Saturation(K) = 1 - (ΔInfo(Kₙ) / ΔInfo(K₁))
+```
+
+Where ΔInfo represents marginal information gain from the nth crawl.
+
+### 3.5 Link Value Prediction
+
+Expected information gain from uncrawled links:
+
+```
+ExpectedGain(l) = Relevance(l, Q) × Novelty(l, K) × Authority(l)
+```
+
+Components:
+- **Relevance**: BM25(preview_text, Q)
+- **Novelty**: 1 - max_similarity(preview, K)
+- **Authority**: f(url_structure, domain_metrics)
+
+## 4. Algorithmic Approach
+
+### 4.1 Progressive Crawling Algorithm
+
+```
+Algorithm: ProgressiveCrawl(start_url, query, θ)
+  K ← ∅
+  crawled ← {start_url}
+  pending ← extract_links(crawl(start_url))
+  
+  while IS(K, Q) < θ and |crawled| < max_pages:
+    candidates ← rank_by_expected_gain(pending, Q, K)
+    if max(ExpectedGain(candidates)) < min_gain:
+      break  // Diminishing returns
+    
+    to_crawl ← top_k(candidates)
+    new_docs ← parallel_crawl(to_crawl)
+    K ← K ∪ new_docs
+    crawled ← crawled ∪ to_crawl
+    pending ← extract_new_links(new_docs) - crawled
+  
+  return K
+```
+
+### 4.2 Stopping Criteria
+
+Crawling terminates when:
+1. IS(K, Q) ≥ θ (sufficient information)
+2. d(IS)/d(crawls) < ε (plateau reached)
+3. |crawled| ≥ max_pages (resource limit)
+4. max(ExpectedGain) < min_gain (no promising links)
+
+## 5. Multi-Strategy Architecture
+
+### 5.1 Strategy Pattern Design
+
+```
+AbstractStrategy
+  ├── StatisticalStrategy (no LLM, no embeddings)
+  ├── EmbeddingStrategy (with semantic similarity)
+  └── LLMStrategy (with language model assistance)
+```
+
+### 5.2 Statistical Strategy
+
+Pure statistical approach using:
+- BM25 for relevance scoring
+- Term frequency analysis for coverage
+- Graph structure for authority
+- No external models required
+
+**Advantages**: Fast, no API costs, works offline
+**Best for**: Technical documentation, specific terminology
+
+### 5.3 Embedding Strategy (Implemented)
+
+Semantic understanding through embeddings:
+- Query expansion into semantic variations
+- Coverage mapping in embedding space
+- Gap-driven link selection
+- Validation-based stopping criteria
+
+**Mathematical Framework**:
+```
+Coverage(K, Q) = mean(max_similarity(q, K) for q in Q_expanded)
+Gap(q) = 1 - max_similarity(q, K)
+LinkScore(l) = Σ(Gap(q) × relevance(l, q)) × (1 - redundancy(l, K))
+```
+
+**Key Parameters**:
+- `embedding_k_exp`: Exponential decay factor for distance-to-score mapping
+- `embedding_coverage_radius`: Distance threshold for query coverage
+- `embedding_min_confidence_threshold`: Minimum relevance threshold
+
+**Advantages**: Semantic understanding, handles ambiguity, detects irrelevance
+**Best for**: Research queries, conceptual topics, diverse content
+
+### 5.4 Progressive Enhancement Path
+
+1. **Level 0**: Statistical only (implemented)
+2. **Level 1**: + Embeddings for semantic similarity (implemented)
+3. **Level 2**: + LLM for query understanding (future)
+
+## 6. Evaluation Methodology
+
+### 6.1 Synthetic Dataset Generation
+
+Using LLM to create evaluation data:
+
+```python
+def generate_synthetic_dataset(domain_url):
+    # 1. Fully crawl domain
+    full_knowledge = exhaustive_crawl(domain_url)
+    
+    # 2. Generate answerable queries
+    queries = llm_generate_queries(full_knowledge)
+    
+    # 3. Create query variations
+    for q in queries:
+        variations = generate_variations(q)  # synonyms, sub/super queries
+    
+    return queries, variations, full_knowledge
+```
+
+### 6.2 Evaluation Metrics
+
+1. **Efficiency**: Information gained / Pages crawled
+2. **Completeness**: Answerable queries / Total queries
+3. **Redundancy**: 1 - (Unique information / Total information)
+4. **Convergence Rate**: Pages to 95% completeness
+
+### 6.3 Ablation Studies
+
+- Impact of each score component (coverage, consistency, saturation)
+- Sensitivity to threshold parameters
+- Performance across different domain types
+
+## 7. Theoretical Properties
+
+### 7.1 Convergence Guarantee
+
+**Theorem**: For finite websites, ProgressiveCrawl converges to IS(K, Q) ≥ θ or exhausts all reachable pages.
+
+**Proof sketch**: IS(K, Q) is monotonically non-decreasing with each crawl, bounded above by 1.
+
+### 7.2 Optimality
+
+Under certain assumptions about link preview accuracy:
+- Expected crawls ≤ 2 × optimal_crawls
+- Approximation ratio improves with preview quality
+
+## 8. Implementation Design
+
+### 8.1 Core Components
+
+1. **CrawlState**: Maintains crawl history and metrics
+2. **AdaptiveConfig**: Configuration parameters
+3. **CrawlStrategy**: Pluggable strategy interface
+4. **AdaptiveCrawler**: Main orchestrator
+
+### 8.2 Integration with Crawl4AI
+
+- Wraps existing AsyncWebCrawler
+- Leverages link preview functionality
+- Maintains backward compatibility
+
+### 8.3 Persistence
+
+Knowledge base serialization for:
+- Resumable crawls
+- Knowledge sharing
+- Offline analysis
+
+## 9. Future Directions
+
+### 9.1 Advanced Scoring
+
+- Temporal information value
+- Multi-query optimization
+- Active learning from user feedback
+
+### 9.2 Distributed Crawling
+
+- Collaborative knowledge building
+- Federated information sufficiency
+
+### 9.3 Domain Adaptation
+
+- Transfer learning across domains
+- Meta-learning for threshold selection
+
+## 10. Conclusion
+
+Progressive crawling with adaptive information foraging provides a principled approach to efficient web information extraction. By combining coverage, consistency, and saturation metrics, we can determine information sufficiency without ground truth labels. The multi-strategy architecture allows graceful enhancement from pure statistical to LLM-assisted approaches based on requirements and resources.
+
+## References
+
+1. Manning, C. D., Raghavan, P., & Schütze, H. (2008). Introduction to Information Retrieval. Cambridge University Press.
+
+2. Robertson, S., & Zaragoza, H. (2009). The Probabilistic Relevance Framework: BM25 and Beyond. Foundations and Trends in Information Retrieval.
+
+3. Pirolli, P., & Card, S. (1999). Information Foraging. Psychological Review, 106(4), 643-675.
+
+4. Dasgupta, S. (2005). Analysis of a greedy active learning strategy. Advances in Neural Information Processing Systems.
+
+## Appendix A: Implementation Pseudocode
+
+```python
+class StatisticalStrategy:
+    def calculate_confidence(self, state):
+        coverage = self.calculate_coverage(state)
+        consistency = self.calculate_consistency(state)
+        saturation = self.calculate_saturation(state)
+        return min(coverage, consistency, saturation)
+    
+    def calculate_coverage(self, state):
+        # BM25-based term coverage
+        term_scores = []
+        for term in state.query.split():
+            df = state.document_frequencies.get(term, 0)
+            idf = self.idf_cache.get(term, 1.0)
+            term_scores.append(log(df + 1) * idf)
+        return mean(term_scores) / max_possible_score
+    
+    def rank_links(self, state):
+        scored_links = []
+        for link in state.pending_links:
+            relevance = self.bm25_score(link.preview_text, state.query)
+            novelty = self.calculate_novelty(link, state.knowledge_base)
+            authority = self.url_authority(link.href)
+            score = relevance * novelty * authority
+            scored_links.append((link, score))
+        return sorted(scored_links, key=lambda x: x[1], reverse=True)
+```
+
+## Appendix B: Evaluation Protocol
+
+1. **Dataset Creation**:
+   - Select diverse domains (documentation, blogs, e-commerce)
+   - Generate 100 queries per domain using LLM
+   - Create query variations (5-10 per query)
+
+2. **Baseline Comparisons**:
+   - BFS crawler (depth-limited)
+   - DFS crawler (depth-limited)
+   - Random crawler
+   - Oracle (knows relevant pages)
+
+3. **Metrics Collection**:
+   - Pages crawled vs query answerability
+   - Time to sufficient confidence
+   - False positive/negative rates
+
+4. **Statistical Analysis**:
+   - ANOVA for strategy comparison
+   - Regression for parameter sensitivity
+   - Bootstrap for confidence intervals

README-first.md

@@ -0,0 +1,809 @@
+# 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper.
+
+<div align="center">
+
+<a href="https://trendshift.io/repositories/11716" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11716" alt="unclecode%2Fcrawl4ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+
+[![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
+[![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members)
+
+[![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai)
+[![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai)](https://pypi.org/project/crawl4ai/)
+[![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
+[![GitHub Sponsors](https://img.shields.io/github/sponsors/unclecode?style=flat&logo=GitHub-Sponsors&label=Sponsors&color=pink)](https://github.com/sponsors/unclecode)
+
+<p align="center">
+    <a href="https://x.com/crawl4ai">
+      <img src="https://img.shields.io/badge/Follow%20on%20X-000000?style=for-the-badge&logo=x&logoColor=white" alt="Follow on X" />
+    </a>
+    <a href="https://www.linkedin.com/company/crawl4ai">
+      <img src="https://img.shields.io/badge/Follow%20on%20LinkedIn-0077B5?style=for-the-badge&logo=linkedin&logoColor=white" alt="Follow on LinkedIn" />
+    </a>
+    <a href="https://discord.gg/jP8KfhDhyN">
+      <img src="https://img.shields.io/badge/Join%20our%20Discord-5865F2?style=for-the-badge&logo=discord&logoColor=white" alt="Join our Discord" />
+    </a>
+  </p>
+</div>
+
+Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for LLMs, AI agents, and data pipelines. Open source, flexible, and built for real-time performance, Crawl4AI empowers developers with unmatched speed, precision, and deployment ease.  
+
+[✨ Check out latest update v0.7.0](#-recent-updates)
+
+🎉 **Version 0.7.0 is now available!** The Adaptive Intelligence Update introduces groundbreaking features: Adaptive Crawling that learns website patterns, Virtual Scroll support for infinite pages, intelligent Link Preview with 3-layer scoring, Async URL Seeder for massive discovery, and significant performance improvements. [Read the release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.0.md)
+
+<details>
+<summary>🤓 <strong>My Personal Story</strong></summary>
+
+My journey with computers started in childhood when my dad, a computer scientist, introduced me to an Amstrad computer. Those early days sparked a fascination with technology, leading me to pursue computer science and specialize in NLP during my postgraduate studies. It was during this time that I first delved into web crawling, building tools to help researchers organize papers and extract information from publications a challenging yet rewarding experience that honed my skills in data extraction.
+
+Fast forward to 2023, I was working on a tool for a project and needed a crawler to convert a webpage into markdown. While exploring solutions, I found one that claimed to be open-source but required creating an account and generating an API token. Worse, it turned out to be a SaaS model charging $16, and its quality didn’t meet my standards. Frustrated, I realized this was a deeper problem. That frustration turned into turbo anger mode, and I decided to build my own solution. In just a few days, I created Crawl4AI. To my surprise, it went viral, earning thousands of GitHub stars and resonating with a global community.
+
+I made Crawl4AI open-source for two reasons. First, it’s my way of giving back to the open-source community that has supported me throughout my career. Second, I believe data should be accessible to everyone, not locked behind paywalls or monopolized by a few. Open access to data lays the foundation for the democratization of AI, a vision where individuals can train their own models and take ownership of their information. This library is the first step in a larger journey to create the best open-source data extraction and generation tool the world has ever seen, built collaboratively by a passionate community.
+
+Thank you to everyone who has supported this project, used it, and shared feedback. Your encouragement motivates me to dream even bigger. Join us, file issues, submit PRs, or spread the word. Together, we can build a tool that truly empowers people to access their own data and reshape the future of AI.
+</details>
+
+## 🧐 Why Crawl4AI?
+
+1. **Built for LLMs**: Creates smart, concise Markdown optimized for RAG and fine-tuning applications.  
+2. **Lightning Fast**: Delivers results faster with real-time, cost-efficient performance.  
+3. **Flexible Browser Control**: Offers session management, proxies, and custom hooks for seamless data access.  
+4. **Heuristic Intelligence**: Uses advanced algorithms for efficient extraction, reducing reliance on costly models.  
+5. **Open Source & Deployable**: Fully open-source with no API keys—ready for Docker and cloud integration.  
+6. **Thriving Community**: Actively maintained by a vibrant community and the #1 trending GitHub repository.
+
+## 🚀 Quick Start 
+
+1. Install Crawl4AI:
+```bash
+# Install the package
+pip install -U crawl4ai
+
+# For pre release versions
+pip install crawl4ai --pre
+
+# Run post-installation setup
+crawl4ai-setup
+
+# Verify your installation
+crawl4ai-doctor
+```
+
+If you encounter any browser-related issues, you can install them manually:
+```bash
+python -m playwright install --with-deps chromium
+```
+
+2. Run a simple web crawl with Python:
+```python
+import asyncio
+from crawl4ai import *
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+        )
+        print(result.markdown)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+3. Or use the new command-line interface:
+```bash
+# Basic crawl with markdown output
+crwl https://www.nbcnews.com/business -o markdown
+
+# Deep crawl with BFS strategy, max 10 pages
+crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10
+
+# Use LLM extraction with a specific question
+crwl https://www.example.com/products -q "Extract all product prices"
+```
+
+## ✨ Features 
+
+<details>
+<summary>📝 <strong>Markdown Generation</strong></summary>
+
+- 🧹 **Clean Markdown**: Generates clean, structured Markdown with accurate formatting.
+- 🎯 **Fit Markdown**: Heuristic-based filtering to remove noise and irrelevant parts for AI-friendly processing.
+- 🔗 **Citations and References**: Converts page links into a numbered reference list with clean citations.
+- 🛠️ **Custom Strategies**: Users can create their own Markdown generation strategies tailored to specific needs.
+- 📚 **BM25 Algorithm**: Employs BM25-based filtering for extracting core information and removing irrelevant content. 
+</details>
+
+<details>
+<summary>📊 <strong>Structured Data Extraction</strong></summary>
+
+- 🤖 **LLM-Driven Extraction**: Supports all LLMs (open-source and proprietary) for structured data extraction.
+- 🧱 **Chunking Strategies**: Implements chunking (topic-based, regex, sentence-level) for targeted content processing.
+- 🌌 **Cosine Similarity**: Find relevant content chunks based on user queries for semantic extraction.
+- 🔎 **CSS-Based Extraction**: Fast schema-based data extraction using XPath and CSS selectors.
+- 🔧 **Schema Definition**: Define custom schemas for extracting structured JSON from repetitive patterns.
+
+</details>
+
+<details>
+<summary>🌐 <strong>Browser Integration</strong></summary>
+
+- 🖥️ **Managed Browser**: Use user-owned browsers with full control, avoiding bot detection.
+- 🔄 **Remote Browser Control**: Connect to Chrome Developer Tools Protocol for remote, large-scale data extraction.
+- 👤 **Browser Profiler**: Create and manage persistent profiles with saved authentication states, cookies, and settings.
+- 🔒 **Session Management**: Preserve browser states and reuse them for multi-step crawling.
+- 🧩 **Proxy Support**: Seamlessly connect to proxies with authentication for secure access.
+- ⚙️ **Full Browser Control**: Modify headers, cookies, user agents, and more for tailored crawling setups.
+- 🌍 **Multi-Browser Support**: Compatible with Chromium, Firefox, and WebKit.
+- 📐 **Dynamic Viewport Adjustment**: Automatically adjusts the browser viewport to match page content, ensuring complete rendering and capturing of all elements.
+
+</details>
+
+<details>
+<summary>🔎 <strong>Crawling & Scraping</strong></summary>
+
+- 🖼️ **Media Support**: Extract images, audio, videos, and responsive image formats like `srcset` and `picture`.
+- 🚀 **Dynamic Crawling**: Execute JS and wait for async or sync for dynamic content extraction.
+- 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis.
+- 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`).
+- 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content.
+- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior.
+- 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches.
+- 📄 **Metadata Extraction**: Retrieve structured metadata from web pages.
+- 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content.
+- 🕵️ **Lazy Load Handling**: Waits for images to fully load, ensuring no content is missed due to lazy loading.
+- 🔄 **Full-Page Scanning**: Simulates scrolling to load and capture all dynamic content, perfect for infinite scroll pages.
+
+</details>
+
+<details>
+<summary>🚀 <strong>Deployment</strong></summary>
+
+- 🐳 **Dockerized Setup**: Optimized Docker image with FastAPI server for easy deployment.
+- 🔑 **Secure Authentication**: Built-in JWT token authentication for API security.
+- 🔄 **API Gateway**: One-click deployment with secure token authentication for API-based workflows.
+- 🌐 **Scalable Architecture**: Designed for mass-scale production and optimized server performance.
+- ☁️ **Cloud Deployment**: Ready-to-deploy configurations for major cloud platforms.
+
+</details>
+
+<details>
+<summary>🎯 <strong>Additional Features</strong></summary>
+
+- 🕶️ **Stealth Mode**: Avoid bot detection by mimicking real users.
+- 🏷️ **Tag-Based Content Extraction**: Refine crawling based on custom tags, headers, or metadata.
+- 🔗 **Link Analysis**: Extract and analyze all links for detailed data exploration.
+- 🛡️ **Error Handling**: Robust error management for seamless execution.
+- 🔐 **CORS & Static Serving**: Supports filesystem-based caching and cross-origin requests.
+- 📖 **Clear Documentation**: Simplified and updated guides for onboarding and advanced usage.
+- 🙌 **Community Recognition**: Acknowledges contributors and pull requests for transparency.
+
+</details>
+
+## Try it Now!
+
+✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing)
+
+✨ Visit our [Documentation Website](https://docs.crawl4ai.com/)
+
+## Installation 🛠️
+
+Crawl4AI offers flexible installation options to suit various use cases. You can install it as a Python package or use Docker.
+
+<details>
+<summary>🐍 <strong>Using pip</strong></summary>
+
+Choose the installation option that best fits your needs:
+
+### Basic Installation
+
+For basic web crawling and scraping tasks:
+
+```bash
+pip install crawl4ai
+crawl4ai-setup # Setup the browser
+```
+
+By default, this will install the asynchronous version of Crawl4AI, using Playwright for web crawling.
+
+👉 **Note**: When you install Crawl4AI, the `crawl4ai-setup` should automatically install and set up Playwright. However, if you encounter any Playwright-related errors, you can manually install it using one of these methods:
+
+1. Through the command line:
+
+   ```bash
+   playwright install
+   ```
+
+2. If the above doesn't work, try this more specific command:
+
+   ```bash
+   python -m playwright install chromium
+   ```
+
+This second method has proven to be more reliable in some cases.
+
+---
+
+### Installation with Synchronous Version
+
+The sync version is deprecated and will be removed in future versions. If you need the synchronous version using Selenium:
+
+```bash
+pip install crawl4ai[sync]
+```
+
+---
+
+### Development Installation
+
+For contributors who plan to modify the source code:
+
+```bash
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+pip install -e .                    # Basic installation in editable mode
+```
+
+Install optional features:
+
+```bash
+pip install -e ".[torch]"           # With PyTorch features
+pip install -e ".[transformer]"     # With Transformer features
+pip install -e ".[cosine]"          # With cosine similarity features
+pip install -e ".[sync]"            # With synchronous crawling (Selenium)
+pip install -e ".[all]"             # Install all optional features
+```
+
+</details>
+
+<details>
+<summary>🐳 <strong>Docker Deployment</strong></summary>
+
+> 🚀 **Now Available!** Our completely redesigned Docker implementation is here! This new solution makes deployment more efficient and seamless than ever.
+
+### New Docker Features
+
+The new Docker implementation includes:
+- **Browser pooling** with page pre-warming for faster response times
+- **Interactive playground** to test and generate request code
+- **MCP integration** for direct connection to AI tools like Claude Code
+- **Comprehensive API endpoints** including HTML extraction, screenshots, PDF generation, and JavaScript execution
+- **Multi-architecture support** with automatic detection (AMD64/ARM64)
+- **Optimized resources** with improved memory management
+
+### Getting Started
+
+```bash
+# Pull and run the latest release candidate
+docker pull unclecode/crawl4ai:0.7.0
+docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:0.7.0
+
+# Visit the playground at http://localhost:11235/playground
+```
+
+For complete documentation, see our [Docker Deployment Guide](https://docs.crawl4ai.com/core/docker-deployment/).
+
+</details>
+
+---
+
+### Quick Test
+
+Run a quick test (works for both Docker options):
+
+```python
+import requests
+
+# Submit a crawl job
+response = requests.post(
+    "http://localhost:11235/crawl",
+    json={"urls": ["https://example.com"], "priority": 10}
+)
+if response.status_code == 200:
+    print("Crawl job submitted successfully.")
+    
+if "results" in response.json():
+    results = response.json()["results"]
+    print("Crawl job completed. Results:")
+    for result in results:
+        print(result)
+else:
+    task_id = response.json()["task_id"]
+    print(f"Crawl job submitted. Task ID:: {task_id}")
+    result = requests.get(f"http://localhost:11235/task/{task_id}")
+```
+
+For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://docs.crawl4ai.com/basic/docker-deployment/).
+
+</details>
+
+
+## 🔬 Advanced Usage Examples 🔬
+
+You can check the project structure in the directory [docs/examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples). Over there, you can find a variety of examples; here, some popular examples are shared.
+
+<details>
+<summary>📝 <strong>Heuristic Markdown Generation with Clean and Fit Markdown</strong></summary>
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def main():
+    browser_config = BrowserConfig(
+        headless=True,  
+        verbose=True,
+    )
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.ENABLED,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed", min_word_threshold=0)
+        ),
+        # markdown_generator=DefaultMarkdownGenerator(
+        #     content_filter=BM25ContentFilter(user_query="WHEN_WE_FOCUS_BASED_ON_A_USER_QUERY", bm25_threshold=1.0)
+        # ),
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://docs.micronaut.io/4.7.6/guide/",
+            config=run_config
+        )
+        print(len(result.markdown.raw_markdown))
+        print(len(result.markdown.fit_markdown))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>🖥️ <strong>Executing JavaScript & Extract Structured Data without LLMs</strong></summary>
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai import JsonCssExtractionStrategy
+import json
+
+async def main():
+    schema = {
+    "name": "KidoCode Courses",
+    "baseSelector": "section.charge-methodology .w-tab-content > div",
+    "fields": [
+        {
+            "name": "section_title",
+            "selector": "h3.heading-50",
+            "type": "text",
+        },
+        {
+            "name": "section_description",
+            "selector": ".charge-content",
+            "type": "text",
+        },
+        {
+            "name": "course_name",
+            "selector": ".text-block-93",
+            "type": "text",
+        },
+        {
+            "name": "course_description",
+            "selector": ".course-content-text",
+            "type": "text",
+        },
+        {
+            "name": "course_icon",
+            "selector": ".image-92",
+            "type": "attribute",
+            "attribute": "src"
+        }
+    }
+}
+
+    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True
+    )
+    run_config = CrawlerRunConfig(
+        extraction_strategy=extraction_strategy,
+        js_code=["""(async () => {const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");for(let tab of tabs) {tab.scrollIntoView();tab.click();await new Promise(r => setTimeout(r, 500));}})();"""],
+        cache_mode=CacheMode.BYPASS
+    )
+        
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            config=run_config
+        )
+
+        companies = json.loads(result.extracted_content)
+        print(f"Successfully extracted {len(companies)} companies")
+        print(json.dumps(companies[0], indent=2))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>📚 <strong>Extracting Structured Data with LLMs</strong></summary>
+
+```python
+import os
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
+from crawl4ai import LLMExtractionStrategy
+from pydantic import BaseModel, Field
+
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(..., description="Fee for output token for the OpenAI model.")
+
+async def main():
+    browser_config = BrowserConfig(verbose=True)
+    run_config = CrawlerRunConfig(
+        word_count_threshold=1,
+        extraction_strategy=LLMExtractionStrategy(
+            # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2
+            # provider="ollama/qwen2", api_token="no-token", 
+            llm_config = LLMConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')), 
+            schema=OpenAIModelFee.schema(),
+            extraction_type="schema",
+            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+            Do not miss any models in the entire content. One extracted model JSON format should look like this: 
+            {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""
+        ),            
+        cache_mode=CacheMode.BYPASS,
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url='https://openai.com/api/pricing/',
+            config=run_config
+        )
+        print(result.extracted_content)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>🤖 <strong>Using Your own Browser with Custom User Profile</strong></summary>
+
+```python
+import os, sys
+from pathlib import Path
+import asyncio, time
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def test_news_crawl():
+    # Create a persistent user data directory
+    user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
+    os.makedirs(user_data_dir, exist_ok=True)
+
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=True,
+        user_data_dir=user_data_dir,
+        use_persistent_context=True,
+    )
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        url = "ADDRESS_OF_A_CHALLENGING_WEBSITE"
+        
+        result = await crawler.arun(
+            url,
+            config=run_config,
+            magic=True,
+        )
+        
+        print(f"Successfully crawled {url}")
+        print(f"Content length: {len(result.markdown)}")
+```
+
+</details>
+
+## ✨ Recent Updates
+
+### Version 0.7.0 Release Highlights - The Adaptive Intelligence Update
+
+- **🧠 Adaptive Crawling**: Your crawler now learns and adapts to website patterns automatically:
+  ```python
+  config = AdaptiveConfig(
+      confidence_threshold=0.7, # Min confidence to stop crawling
+      max_depth=5, # Maximum crawl depth
+      max_pages=20, # Maximum number of pages to crawl
+      strategy="statistical"
+  )
+  
+  async with AsyncWebCrawler() as crawler:
+      adaptive_crawler = AdaptiveCrawler(crawler, config)
+      state = await adaptive_crawler.digest(
+          start_url="https://news.example.com",
+          query="latest news content"
+      )
+  # Crawler learns patterns and improves extraction over time
+  ```
+
+- **🌊 Virtual Scroll Support**: Complete content extraction from infinite scroll pages:
+  ```python
+  scroll_config = VirtualScrollConfig(
+      container_selector="[data-testid='feed']",
+      scroll_count=20,
+      scroll_by="container_height",
+      wait_after_scroll=1.0
+  )
+  
+  result = await crawler.arun(url, config=CrawlerRunConfig(
+      virtual_scroll_config=scroll_config
+  ))
+  ```
+
+- **🔗 Intelligent Link Analysis**: 3-layer scoring system for smart link prioritization:
+  ```python
+  link_config = LinkPreviewConfig(
+      query="machine learning tutorials",
+      score_threshold=0.3,
+      concurrent_requests=10
+  )
+  
+  result = await crawler.arun(url, config=CrawlerRunConfig(
+      link_preview_config=link_config,
+      score_links=True
+  ))
+  # Links ranked by relevance and quality
+  ```
+
+- **🎣 Async URL Seeder**: Discover thousands of URLs in seconds:
+  ```python
+  seeder = AsyncUrlSeeder(SeedingConfig(
+      source="sitemap+cc",
+      pattern="*/blog/*",
+      query="python tutorials",
+      score_threshold=0.4
+  ))
+  
+  urls = await seeder.discover("https://example.com")
+  ```
+
+- **⚡ Performance Boost**: Up to 3x faster with optimized resource handling and memory efficiency
+
+Read the full details in our [0.7.0 Release Notes](https://docs.crawl4ai.com/blog/release-v0.7.0) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
+
+## Version Numbering in Crawl4AI
+
+Crawl4AI follows standard Python version numbering conventions (PEP 440) to help users understand the stability and features of each release.
+
+### Version Numbers Explained
+
+Our version numbers follow this pattern: `MAJOR.MINOR.PATCH` (e.g., 0.4.3)
+
+#### Pre-release Versions
+We use different suffixes to indicate development stages:
+
+- `dev` (0.4.3dev1): Development versions, unstable
+- `a` (0.4.3a1): Alpha releases, experimental features
+- `b` (0.4.3b1): Beta releases, feature complete but needs testing
+- `rc` (0.4.3): Release candidates, potential final version
+
+#### Installation
+- Regular installation (stable version):
+  ```bash
+  pip install -U crawl4ai
+  ```
+
+- Install pre-release versions:
+  ```bash
+  pip install crawl4ai --pre
+  ```
+
+- Install specific version:
+  ```bash
+  pip install crawl4ai==0.4.3b1
+  ```
+
+#### Why Pre-releases?
+We use pre-releases to:
+- Test new features in real-world scenarios
+- Gather feedback before final releases
+- Ensure stability for production users
+- Allow early adopters to try new features
+
+For production environments, we recommend using the stable version. For testing new features, you can opt-in to pre-releases using the `--pre` flag.
+
+## 📖 Documentation & Roadmap 
+
+> 🚨 **Documentation Update Alert**: We're undertaking a major documentation overhaul next week to reflect recent updates and improvements. Stay tuned for a more comprehensive and up-to-date guide!
+
+For current documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://docs.crawl4ai.com/).
+
+To check our development plans and upcoming features, visit our [Roadmap](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md).
+
+<details>
+<summary>📈 <strong>Development TODOs</strong></summary>
+
+- [x] 0. Graph Crawler: Smart website traversal using graph search algorithms for comprehensive nested page extraction
+- [ ] 1. Question-Based Crawler: Natural language driven web discovery and content extraction
+- [ ] 2. Knowledge-Optimal Crawler: Smart crawling that maximizes knowledge while minimizing data extraction
+- [ ] 3. Agentic Crawler: Autonomous system for complex multi-step crawling operations
+- [ ] 4. Automated Schema Generator: Convert natural language to extraction schemas
+- [ ] 5. Domain-Specific Scrapers: Pre-configured extractors for common platforms (academic, e-commerce)
+- [ ] 6. Web Embedding Index: Semantic search infrastructure for crawled content
+- [ ] 7. Interactive Playground: Web UI for testing, comparing strategies with AI assistance
+- [ ] 8. Performance Monitor: Real-time insights into crawler operations
+- [ ] 9. Cloud Integration: One-click deployment solutions across cloud providers
+- [ ] 10. Sponsorship Program: Structured support system with tiered benefits
+- [ ] 11. Educational Content: "How to Crawl" video series and interactive tutorials
+
+</details>
+
+## 🤝 Contributing 
+
+We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information.
+
+I'll help modify the license section with badges. For the halftone effect, here's a version with it:
+
+Here's the updated license section:
+
+## 📄 License & Attribution
+
+This project is licensed under the Apache License 2.0, attribution is recommended via the badges below. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
+
+### Attribution Requirements
+When using Crawl4AI, you must include one of the following attribution methods:
+
+#### 1. Badge Attribution (Recommended)
+Add one of these badges to your README, documentation, or website:
+
+| Theme | Badge |
+|-------|-------|
+| **Disco Theme (Animated)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Night Theme (Dark with Neon)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Dark Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Light Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+ 
+
+HTML code for adding the badges:
+```html
+<!-- Disco Theme (Animated) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Night Theme (Dark with Neon) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Dark Theme (Classic) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Light Theme (Classic) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Simple Shield Badge -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://img.shields.io/badge/Powered%20by-Crawl4AI-blue?style=flat-square" alt="Powered by Crawl4AI"/>
+</a>
+```
+
+#### 2. Text Attribution
+Add this line to your documentation:
+```
+This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
+```
+
+## 📚 Citation
+
+If you use Crawl4AI in your research or project, please cite:
+
+```bibtex
+@software{crawl4ai2024,
+  author = {UncleCode},
+  title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper},
+  year = {2024},
+  publisher = {GitHub},
+  journal = {GitHub Repository},
+  howpublished = {\url{https://github.com/unclecode/crawl4ai}},
+  commit = {Please use the commit hash you're working with}
+}
+```
+
+Text citation format:
+```
+UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software]. 
+GitHub. https://github.com/unclecode/crawl4ai
+```
+
+## 📧 Contact 
+
+For questions, suggestions, or feedback, feel free to reach out:
+
+- GitHub: [unclecode](https://github.com/unclecode)
+- Twitter: [@unclecode](https://twitter.com/unclecode)
+- Website: [crawl4ai.com](https://crawl4ai.com)
+
+Happy Crawling! 🕸️🚀
+
+## 💖 Support Crawl4AI
+
+> 🎉 **Sponsorship Program Just Launched!** Be among the first 50 **Founding Sponsors** and get permanent recognition in our Hall of Fame!
+
+Crawl4AI is the #1 trending open-source web crawler with 51K+ stars. Your support ensures we stay independent, innovative, and free forever.
+
+<div align="center">
+
+[![Become a Sponsor](https://img.shields.io/badge/Become%20a%20Sponsor-pink?style=for-the-badge&logo=github-sponsors&logoColor=white)](https://github.com/sponsors/unclecode)
+[![Current Sponsors](https://img.shields.io/github/sponsors/unclecode?style=for-the-badge&logo=github&label=Current%20Sponsors&color=green)](https://github.com/sponsors/unclecode)
+
+</div>
+
+### 🤝 Sponsorship Tiers
+
+- **🌱 Believer ($5/mo)**: Join the movement for data democratization
+- **🚀 Builder ($50/mo)**: Get priority support and early feature access  
+- **💼 Growing Team ($500/mo)**: Bi-weekly syncs and optimization help
+- **🏢 Data Infrastructure Partner ($2000/mo)**: Full partnership with dedicated support
+
+**Why sponsor?** Every tier includes real benefits. No more rate-limited APIs. Own your data pipeline. Build data sovereignty together.
+
+[View All Tiers & Benefits →](https://github.com/sponsors/unclecode)
+
+### 🏆 Our Sponsors
+
+#### 👑 Founding Sponsors (First 50)
+*Be part of history - [Become a Founding Sponsor](https://github.com/sponsors/unclecode)*
+
+<!-- Founding sponsors will be permanently recognized here -->
+
+#### Current Sponsors
+Thank you to all our sponsors who make this project possible!
+
+<!-- Sponsors will be automatically added here -->
+
+## 🗾 Mission
+
+Our mission is to unlock the value of personal and enterprise data by transforming digital footprints into structured, tradeable assets. Crawl4AI empowers individuals and organizations with open-source tools to extract and structure data, fostering a shared data economy.  
+
+We envision a future where AI is powered by real human knowledge, ensuring data creators directly benefit from their contributions. By democratizing data and enabling ethical sharing, we are laying the foundation for authentic AI advancement.
+
+<details>
+<summary>🔑 <strong>Key Opportunities</strong></summary>
+ 
+- **Data Capitalization**: Transform digital footprints into measurable, valuable assets.  
+- **Authentic AI Data**: Provide AI systems with real human insights.  
+- **Shared Economy**: Create a fair data marketplace that benefits data creators.  
+
+</details>
+
+<details>
+<summary>🚀 <strong>Development Pathway</strong></summary>
+
+1. **Open-Source Tools**: Community-driven platforms for transparent data extraction.  
+2. **Digital Asset Structuring**: Tools to organize and value digital knowledge.  
+3. **Ethical Data Marketplace**: A secure, fair platform for exchanging structured data.  
+
+For more details, see our [full mission statement](./MISSION.md).
+</details>
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

README.md

@@ -10,41 +10,48 @@
 [![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai)
 [![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai)](https://pypi.org/project/crawl4ai/)
 [![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
+[![GitHub Sponsors](https://img.shields.io/github/sponsors/unclecode?style=flat&logo=GitHub-Sponsors&label=Sponsors&color=pink)](https://github.com/sponsors/unclecode)
 
-<!-- [![Documentation Status](https://readthedocs.org/projects/crawl4ai/badge/?version=latest)](https://crawl4ai.readthedocs.io/) -->
-[![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
-[![Security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
-[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md)
-
+<p align="center">
+    <a href="https://x.com/crawl4ai">
+      <img src="https://img.shields.io/badge/Follow%20on%20X-000000?style=for-the-badge&logo=x&logoColor=white" alt="Follow on X" />
+    </a>
+    <a href="https://www.linkedin.com/company/crawl4ai">
+      <img src="https://img.shields.io/badge/Follow%20on%20LinkedIn-0077B5?style=for-the-badge&logo=linkedin&logoColor=white" alt="Follow on LinkedIn" />
+    </a>
+    <a href="https://discord.gg/jP8KfhDhyN">
+      <img src="https://img.shields.io/badge/Join%20our%20Discord-5865F2?style=for-the-badge&logo=discord&logoColor=white" alt="Join our Discord" />
+    </a>
+  </p>
 </div>
 
-Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for LLMs, AI agents, and data pipelines. Open source, flexible, and built for real-time performance, Crawl4AI empowers developers with unmatched speed, precision, and deployment ease.  
+Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
 
-[✨ Check out latest update v0.5.0](#-recent-updates)
+[✨ Check out latest update v0.7.0](#-recent-updates)
 
-🎉 **Version 0.5.0 is out!** This major release introduces Deep Crawling with BFS/DFS/BestFirst strategies, Memory-Adaptive Dispatcher, Multiple Crawling Strategies (Playwright and HTTP), Docker Deployment with FastAPI, Command-Line Interface (CLI), and more! [Read the release notes →](https://docs.crawl4ai.com/blog)
+✨ New in v0.7.0, Adaptive Crawling, Virtual Scroll, Link Preview scoring, Async URL Seeder, big performance gains. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.0.md)
 
 <details>
-<summary>🤓 <strong>My Personal Story</strong></summary>
+  <summary>🤓 <strong>My Personal Story</strong></summary>
 
-My journey with computers started in childhood when my dad, a computer scientist, introduced me to an Amstrad computer. Those early days sparked a fascination with technology, leading me to pursue computer science and specialize in NLP during my postgraduate studies. It was during this time that I first delved into web crawling, building tools to help researchers organize papers and extract information from publications a challenging yet rewarding experience that honed my skills in data extraction.
+I grew up on an Amstrad, thanks to my dad, and never stopped building. In grad school I specialized in NLP and built crawlers for research. That’s where I learned how much extraction matters.
 
-Fast forward to 2023, I was working on a tool for a project and needed a crawler to convert a webpage into markdown. While exploring solutions, I found one that claimed to be open-source but required creating an account and generating an API token. Worse, it turned out to be a SaaS model charging $16, and its quality didn’t meet my standards. Frustrated, I realized this was a deeper problem. That frustration turned into turbo anger mode, and I decided to build my own solution. In just a few days, I created Crawl4AI. To my surprise, it went viral, earning thousands of GitHub stars and resonating with a global community.
+In 2023, I needed web-to-Markdown. The “open source” option wanted an account, API token, and $16, and still under-delivered. I went turbo anger mode, built Crawl4AI in days, and it went viral. Now it’s the most-starred crawler on GitHub.
 
-I made Crawl4AI open-source for two reasons. First, it’s my way of giving back to the open-source community that has supported me throughout my career. Second, I believe data should be accessible to everyone, not locked behind paywalls or monopolized by a few. Open access to data lays the foundation for the democratization of AI, a vision where individuals can train their own models and take ownership of their information. This library is the first step in a larger journey to create the best open-source data extraction and generation tool the world has ever seen, built collaboratively by a passionate community.
-
-Thank you to everyone who has supported this project, used it, and shared feedback. Your encouragement motivates me to dream even bigger. Join us, file issues, submit PRs, or spread the word. Together, we can build a tool that truly empowers people to access their own data and reshape the future of AI.
+I made it open source for **availability**, anyone can use it without a gate. Now I’m building the platform for **affordability**, anyone can run serious crawls without breaking the bank. If that resonates, join in, send feedback, or just crawl something amazing.
 </details>
 
-## 🧐 Why Crawl4AI?
 
-1. **Built for LLMs**: Creates smart, concise Markdown optimized for RAG and fine-tuning applications.  
-2. **Lightning Fast**: Delivers results 6x faster with real-time, cost-efficient performance.  
-3. **Flexible Browser Control**: Offers session management, proxies, and custom hooks for seamless data access.  
-4. **Heuristic Intelligence**: Uses advanced algorithms for efficient extraction, reducing reliance on costly models.  
-5. **Open Source & Deployable**: Fully open-source with no API keys—ready for Docker and cloud integration.  
-6. **Thriving Community**: Actively maintained by a vibrant community and the #1 trending GitHub repository.
+<details>
+  <summary>Why developers pick Crawl4AI</summary>
+
+- **LLM ready output**, smart Markdown with headings, tables, code, citation hints
+- **Fast in practice**, async browser pool, caching, minimal hops
+- **Full control**, sessions, proxies, cookies, user scripts, hooks
+- **Adaptive intelligence**, learns site patterns, explores only what matters
+- **Deploy anywhere**, zero keys, CLI and Docker, cloud friendly
+</details>
+
 
 ## 🚀 Quick Start 
 
@@ -96,6 +103,33 @@ crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10
 crwl https://www.example.com/products -q "Extract all product prices"
 ```
 
+## 💖 Support Crawl4AI
+
+> 🎉 **Sponsorship Program Now Open!** After powering 51K+ developers and 1 year of growth, Crawl4AI is launching dedicated support for **startups** and **enterprises**. Be among the first 50 **Founding Sponsors** for permanent recognition in our Hall of Fame.
+
+Crawl4AI is the #1 trending open-source web crawler on GitHub. Your support keeps it independent, innovative, and free for the community — while giving you direct access to premium benefits.
+
+<div align="">
+  
+[![Become a Sponsor](https://img.shields.io/badge/Become%20a%20Sponsor-pink?style=for-the-badge&logo=github-sponsors&logoColor=white)](https://github.com/sponsors/unclecode)  
+[![Current Sponsors](https://img.shields.io/github/sponsors/unclecode?style=for-the-badge&logo=github&label=Current%20Sponsors&color=green)](https://github.com/sponsors/unclecode)
+
+</div>
+
+### 🤝 Sponsorship Tiers
+
+- **🌱 Believer ($5/mo)** — Join the movement for data democratization  
+- **🚀 Builder ($50/mo)** — Priority support & early access to features  
+- **💼 Growing Team ($500/mo)** — Bi-weekly syncs & optimization help  
+- **🏢 Data Infrastructure Partner ($2000/mo)** — Full partnership with dedicated support  
+  *Custom arrangements available - see [SPONSORS.md](SPONSORS.md) for details & contact*
+
+**Why sponsor?**  
+No rate-limited APIs. No lock-in. Build and own your data pipeline with direct guidance from the creator of Crawl4AI.
+
+[See All Tiers & Benefits →](https://github.com/sponsors/unclecode)
+
+
 ## ✨ Features 
 
 <details>
@@ -253,28 +287,27 @@ pip install -e ".[all]"             # Install all optional features
 <details>
 <summary>🐳 <strong>Docker Deployment</strong></summary>
 
-> 🚀 **Major Changes Coming!** We're developing a completely new Docker implementation that will make deployment even more efficient and seamless. The current Docker setup is being deprecated in favor of this new solution.
+> 🚀 **Now Available!** Our completely redesigned Docker implementation is here! This new solution makes deployment more efficient and seamless than ever.
 
-### Current Docker Support
+### New Docker Features
 
-The existing Docker implementation is being deprecated and will be replaced soon. If you still need to use Docker with the current version:
+The new Docker implementation includes:
+- **Browser pooling** with page pre-warming for faster response times
+- **Interactive playground** to test and generate request code
+- **MCP integration** for direct connection to AI tools like Claude Code
+- **Comprehensive API endpoints** including HTML extraction, screenshots, PDF generation, and JavaScript execution
+- **Multi-architecture support** with automatic detection (AMD64/ARM64)
+- **Optimized resources** with improved memory management
 
-- 📚 [Deprecated Docker Setup](./docs/deprecated/docker-deployment.md) - Instructions for the current Docker implementation
-- ⚠️ Note: This setup will be replaced in the next major release
+### Getting Started
 
-### What's Coming Next?
+```bash
+# Pull and run the latest release candidate
+docker pull unclecode/crawl4ai:0.7.0
+docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:0.7.0
 
-Our new Docker implementation will bring:
-- Improved performance and resource efficiency
-- Streamlined deployment process
-- Better integration with Crawl4AI features
-- Enhanced scalability options
-
-Stay connected with our [GitHub repository](https://github.com/unclecode/crawl4ai) for updates!
-
-</details>
-
----
+# Visit the playground at http://localhost:11235/playground
+```
 
 ### Quick Test
 
@@ -286,22 +319,31 @@ import requests
 # Submit a crawl job
 response = requests.post(
     "http://localhost:11235/crawl",
-    json={"urls": "https://example.com", "priority": 10}
+    json={"urls": ["https://example.com"], "priority": 10}
 )
-task_id = response.json()["task_id"]
-
-# Continue polling until the task is complete (status="completed")
-result = requests.get(f"http://localhost:11235/task/{task_id}")
+if response.status_code == 200:
+    print("Crawl job submitted successfully.")
+    
+if "results" in response.json():
+    results = response.json()["results"]
+    print("Crawl job completed. Results:")
+    for result in results:
+        print(result)
+else:
+    task_id = response.json()["task_id"]
+    print(f"Crawl job submitted. Task ID:: {task_id}")
+    result = requests.get(f"http://localhost:11235/task/{task_id}")
 ```
 
 For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://docs.crawl4ai.com/basic/docker-deployment/).
 
 </details>
 
+---
 
 ## 🔬 Advanced Usage Examples 🔬
 
-You can check the project structure in the directory [https://github.com/unclecode/crawl4ai/docs/examples](docs/examples). Over there, you can find a variety of examples; here, some popular examples are shared.
+You can check the project structure in the directory [docs/examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples). Over there, you can find a variety of examples; here, some popular examples are shared.
 
 <details>
 <summary>📝 <strong>Heuristic Markdown Generation with Clean and Fit Markdown</strong></summary>
@@ -347,7 +389,7 @@ if __name__ == "__main__":
 ```python
 import asyncio
 from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai import JsonCssExtractionStrategy
 import json
 
 async def main():
@@ -421,7 +463,7 @@ if __name__ == "__main__":
 import os
 import asyncio
 from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
-from crawl4ai.extraction_strategy import LLMExtractionStrategy
+from crawl4ai import LLMExtractionStrategy
 from pydantic import BaseModel, Field
 
 class OpenAIModelFee(BaseModel):
@@ -460,7 +502,7 @@ if __name__ == "__main__":
 </details>
 
 <details>
-<summary>🤖 <strong>Using You own Browser with Custom User Profile</strong></summary>
+<summary>🤖 <strong>Using Your own Browser with Custom User Profile</strong></summary>
 
 ```python
 import os, sys
@@ -500,37 +542,77 @@ async def test_news_crawl():
 
 ## ✨ Recent Updates
 
-### Version 0.5.0 Major Release Highlights
+### Version 0.7.0 Release Highlights - The Adaptive Intelligence Update
 
--   **🚀 Deep Crawling System**: Explore websites beyond initial URLs with three strategies:
-    -   **BFS Strategy**: Breadth-first search explores websites level by level
-    -   **DFS Strategy**: Depth-first search explores each branch deeply before backtracking
-    -   **BestFirst Strategy**: Uses scoring functions to prioritize which URLs to crawl next
-    -   **Page Limiting**: Control the maximum number of pages to crawl with `max_pages` parameter
-    -   **Score Thresholds**: Filter URLs based on relevance scores
--   **⚡ Memory-Adaptive Dispatcher**: Dynamically adjusts concurrency based on system memory with built-in rate limiting
--   **🔄 Multiple Crawling Strategies**:
-    -   **AsyncPlaywrightCrawlerStrategy**: Browser-based crawling with JavaScript support (Default)
-    -   **AsyncHTTPCrawlerStrategy**: Fast, lightweight HTTP-only crawler for simple tasks
--   **🐳 Docker Deployment**: Easy deployment with FastAPI server and streaming/non-streaming endpoints
--   **💻 Command-Line Interface**: New `crwl` CLI provides convenient terminal access to all features with intuitive commands and configuration options
--   **👤 Browser Profiler**: Create and manage persistent browser profiles to save authentication states, cookies, and settings for seamless crawling of protected content
--   **🧠 Crawl4AI Coding Assistant**: AI-powered coding assistant to answer your question for Crawl4ai, and generate proper code for crawling.
--   **🏎️ LXML Scraping Mode**: Fast HTML parsing using the `lxml` library for improved performance
--   **🌐 Proxy Rotation**: Built-in support for proxy switching with `RoundRobinProxyStrategy`
--   **🤖 LLM Content Filter**: Intelligent markdown generation using LLMs
--   **📄 PDF Processing**: Extract text, images, and metadata from PDF files
--   **🔗 URL Redirection Tracking**: Automatically follow and record HTTP redirects
--   **🤖 LLM Schema Generation**: Easily create extraction schemas with LLM assistance
--   **🔍 robots.txt Compliance**: Respect website crawling rules
+- **🧠 Adaptive Crawling**: Your crawler now learns and adapts to website patterns automatically:
+  ```python
+  config = AdaptiveConfig(
+      confidence_threshold=0.7, # Min confidence to stop crawling
+      max_depth=5, # Maximum crawl depth
+      max_pages=20, # Maximum number of pages to crawl
+      strategy="statistical"
+  )
+  
+  async with AsyncWebCrawler() as crawler:
+      adaptive_crawler = AdaptiveCrawler(crawler, config)
+      state = await adaptive_crawler.digest(
+          start_url="https://news.example.com",
+          query="latest news content"
+      )
+  # Crawler learns patterns and improves extraction over time
+  ```
 
-Read the full details in our [0.5.0 Release Notes](https://docs.crawl4ai.com/blog/releases/0.5.0.html) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
+- **🌊 Virtual Scroll Support**: Complete content extraction from infinite scroll pages:
+  ```python
+  scroll_config = VirtualScrollConfig(
+      container_selector="[data-testid='feed']",
+      scroll_count=20,
+      scroll_by="container_height",
+      wait_after_scroll=1.0
+  )
+  
+  result = await crawler.arun(url, config=CrawlerRunConfig(
+      virtual_scroll_config=scroll_config
+  ))
+  ```
+
+- **🔗 Intelligent Link Analysis**: 3-layer scoring system for smart link prioritization:
+  ```python
+  link_config = LinkPreviewConfig(
+      query="machine learning tutorials",
+      score_threshold=0.3,
+      concurrent_requests=10
+  )
+  
+  result = await crawler.arun(url, config=CrawlerRunConfig(
+      link_preview_config=link_config,
+      score_links=True
+  ))
+  # Links ranked by relevance and quality
+  ```
+
+- **🎣 Async URL Seeder**: Discover thousands of URLs in seconds:
+  ```python
+  seeder = AsyncUrlSeeder(SeedingConfig(
+      source="sitemap+cc",
+      pattern="*/blog/*",
+      query="python tutorials",
+      score_threshold=0.4
+  ))
+  
+  urls = await seeder.discover("https://example.com")
+  ```
+
+- **⚡ Performance Boost**: Up to 3x faster with optimized resource handling and memory efficiency
+
+Read the full details in our [0.7.0 Release Notes](https://docs.crawl4ai.com/blog/release-v0.7.0) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
 
 ## Version Numbering in Crawl4AI
 
 Crawl4AI follows standard Python version numbering conventions (PEP 440) to help users understand the stability and features of each release.
 
-### Version Numbers Explained
+<details>
+<summary>📈 <strong>Version Numbers Explained</strong></summary>
 
 Our version numbers follow this pattern: `MAJOR.MINOR.PATCH` (e.g., 0.4.3)
 
@@ -540,7 +622,7 @@ We use different suffixes to indicate development stages:
 - `dev` (0.4.3dev1): Development versions, unstable
 - `a` (0.4.3a1): Alpha releases, experimental features
 - `b` (0.4.3b1): Beta releases, feature complete but needs testing
-- `rc` (0.4.3rc1): Release candidates, potential final version
+- `rc` (0.4.3): Release candidates, potential final version
 
 #### Installation
 - Regular installation (stable version):
@@ -567,6 +649,8 @@ We use pre-releases to:
 
 For production environments, we recommend using the stable version. For testing new features, you can opt-in to pre-releases using the `--pre` flag.
 
+</details>
+
 ## 📖 Documentation & Roadmap 
 
 > 🚨 **Documentation Update Alert**: We're undertaking a major documentation overhaul next week to reflect recent updates and improvements. Stay tuned for a more comprehensive and up-to-date guide!
@@ -579,16 +663,16 @@ To check our development plans and upcoming features, visit our [Roadmap](https:
 <summary>📈 <strong>Development TODOs</strong></summary>
 
 - [x] 0. Graph Crawler: Smart website traversal using graph search algorithms for comprehensive nested page extraction
-- [ ] 1. Question-Based Crawler: Natural language driven web discovery and content extraction
-- [ ] 2. Knowledge-Optimal Crawler: Smart crawling that maximizes knowledge while minimizing data extraction
-- [ ] 3. Agentic Crawler: Autonomous system for complex multi-step crawling operations
-- [ ] 4. Automated Schema Generator: Convert natural language to extraction schemas
-- [ ] 5. Domain-Specific Scrapers: Pre-configured extractors for common platforms (academic, e-commerce)
-- [ ] 6. Web Embedding Index: Semantic search infrastructure for crawled content
-- [ ] 7. Interactive Playground: Web UI for testing, comparing strategies with AI assistance
-- [ ] 8. Performance Monitor: Real-time insights into crawler operations
+- [x] 1. Question-Based Crawler: Natural language driven web discovery and content extraction
+- [x] 2. Knowledge-Optimal Crawler: Smart crawling that maximizes knowledge while minimizing data extraction
+- [x] 3. Agentic Crawler: Autonomous system for complex multi-step crawling operations
+- [x] 4. Automated Schema Generator: Convert natural language to extraction schemas
+- [x] 5. Domain-Specific Scrapers: Pre-configured extractors for common platforms (academic, e-commerce)
+- [x] 6. Web Embedding Index: Semantic search infrastructure for crawled content
+- [x] 7. Interactive Playground: Web UI for testing, comparing strategies with AI assistance
+- [x] 8. Performance Monitor: Real-time insights into crawler operations
 - [ ] 9. Cloud Integration: One-click deployment solutions across cloud providers
-- [ ] 10. Sponsorship Program: Structured support system with tiered benefits
+- [x] 10. Sponsorship Program: Structured support system with tiered benefits
 - [ ] 11. Educational Content: "How to Crawl" video series and interactive tutorials
 
 </details>
@@ -603,12 +687,13 @@ Here's the updated license section:
 
 ## 📄 License & Attribution
 
-This project is licensed under the Apache License 2.0 with a required attribution clause. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
+This project is licensed under the Apache License 2.0, attribution is recommended via the badges below. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
 
 ### Attribution Requirements
 When using Crawl4AI, you must include one of the following attribution methods:
 
-#### 1. Badge Attribution (Recommended)
+<details>
+<summary>📈 <strong>1. Badge Attribution (Recommended)</strong></summary>
 Add one of these badges to your README, documentation, or website:
 
 | Theme | Badge |
@@ -647,11 +732,15 @@ HTML code for adding the badges:
 </a>
 ```
 
-#### 2. Text Attribution
+</details>
+
+<details>
+<summary>📖 <strong>2. Text Attribution</strong></summary>
 Add this line to your documentation:
 ```
 This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
 ```
+</details>
 
 ## 📚 Citation
 

SPONSORS.md

@@ -0,0 +1,65 @@
+# 💖 Sponsors & Supporters
+
+Thank you to everyone supporting Crawl4AI! Your sponsorship helps keep this project open-source and actively maintained.
+
+## 👑 Founding Sponsors
+*The first 50 sponsors who believed in our vision - permanently recognized*
+
+<!-- Founding sponsors will be listed here with special recognition -->
+🎉 **Become a Founding Sponsor!** Only [X/50] spots remaining! [Join now →](https://github.com/sponsors/unclecode)
+
+---
+
+## 🏢 Data Infrastructure Partners ($2000/month)
+*These organizations are building their data sovereignty with Crawl4AI at the core*
+
+<!-- Data Infrastructure Partners will be listed here -->
+*Be the first Data Infrastructure Partner! [Join us →](https://github.com/sponsors/unclecode)*
+
+---
+
+## 💼 Growing Teams ($500/month)
+*Teams scaling their data extraction with Crawl4AI*
+
+<!-- Growing Teams will be listed here -->
+*Your team could be here! [Become a sponsor →](https://github.com/sponsors/unclecode)*
+
+---
+
+## 🚀 Builders ($50/month)
+*Developers and entrepreneurs building with Crawl4AI*
+
+<!-- Builders will be listed here -->
+*Join the builders! [Start sponsoring →](https://github.com/sponsors/unclecode)*
+
+---
+
+## 🌱 Believers ($5/month)
+*The community supporting data democratization*
+
+<!-- Believers will be listed here -->
+*Thank you to all our community believers!*
+
+---
+
+## 🤝 Want to Sponsor?
+
+Crawl4AI is the #1 trending open-source web crawler. We're building the future of data extraction - where organizations own their data pipelines instead of relying on rate-limited APIs.
+
+### Available Sponsorship Tiers:
+- **🌱 Believer** ($5/mo) - Support the movement
+- **🚀 Builder** ($50/mo) - Priority support & early access
+- **💼 Growing Team** ($500/mo) - Bi-weekly syncs & optimization
+- **🏢 Data Infrastructure Partner** ($2000/mo) - Full partnership & dedicated support
+
+[View all tiers and benefits →](https://github.com/sponsors/unclecode)
+
+### Enterprise & Custom Partnerships
+
+Building data extraction at scale? Need dedicated support or infrastructure? Let's talk about a custom partnership.
+
+📧 Contact: [hello@crawl4ai.com](mailto:hello@crawl4ai.com) | 📅 [Schedule a call](https://calendar.app.google/rEpvi2UBgUQjWHfJ9)
+
+---
+
+*This list is updated regularly. Sponsors at $50+ tiers can submit their logos via [hello@crawl4ai.com](mailto:hello@crawl4ai.com)*

crawl4ai/__init__.py

@@ -2,18 +2,13 @@
 import warnings
 
 from .async_webcrawler import AsyncWebCrawler, CacheMode
-from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig, LLMConfig
-
-from .pipeline.pipeline import (
-    Pipeline,
-    create_pipeline,
-)
-from .pipeline.crawler import Crawler
+# MODIFIED: Add SeedingConfig and VirtualScrollConfig here
+from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig, LLMConfig, ProxyConfig, GeolocationConfig, SeedingConfig, VirtualScrollConfig, LinkPreviewConfig, MatchMode
 
 from .content_scraping_strategy import (
     ContentScrapingStrategy,
-    WebScrapingStrategy,
     LXMLWebScrapingStrategy,
+    WebScrapingStrategy,  # Backward compatibility alias
 )
 from .async_logger import (
     AsyncLoggerBase,
@@ -29,7 +24,8 @@ from .extraction_strategy import (
     CosineStrategy,
     JsonCssExtractionStrategy,
     JsonXPathExtractionStrategy,
-    JsonLxmlExtractionStrategy
+    JsonLxmlExtractionStrategy,
+    RegexExtractionStrategy
 )
 from .chunking_strategy import ChunkingStrategy, RegexChunking
 from .markdown_generation_strategy import DefaultMarkdownGenerator
@@ -41,6 +37,7 @@ from .content_filter_strategy import (
 )
 from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
 from .components.crawler_monitor import CrawlerMonitor
+from .link_preview import LinkPreview
 from .async_dispatcher import (
     MemoryAdaptiveDispatcher,
     SemaphoreDispatcher,
@@ -70,20 +67,57 @@ from .deep_crawling import (
     DFSDeepCrawlStrategy,
     DeepCrawlDecorator,
 )
+# NEW: Import AsyncUrlSeeder
+from .async_url_seeder import AsyncUrlSeeder
+# Adaptive Crawler
+from .adaptive_crawler import (
+    AdaptiveCrawler,
+    AdaptiveConfig,
+    CrawlState,
+    CrawlStrategy,
+    StatisticalStrategy
+)
 
-from .async_crawler_strategy import AsyncPlaywrightCrawlerStrategy, AsyncHTTPCrawlerStrategy
+# C4A Script Language Support
+from .script import (
+    compile as c4a_compile,
+    validate as c4a_validate,
+    compile_file as c4a_compile_file,
+    CompilationResult,
+    ValidationResult,
+    ErrorDetail
+)
+
+# Browser Adapters
+from .browser_adapter import (
+    BrowserAdapter,
+    PlaywrightAdapter,
+    UndetectedAdapter
+)
+
+from .utils import (
+    start_colab_display_server,
+    setup_colab_environment
+)
 
 __all__ = [
-    "Pipeline",
-    "AsyncPlaywrightCrawlerStrategy",
-    "AsyncHTTPCrawlerStrategy",
-    "create_pipeline",
-    "Crawler",
     "AsyncLoggerBase",
     "AsyncLogger",
     "AsyncWebCrawler",
     "BrowserProfiler",
     "LLMConfig",
+    "GeolocationConfig",
+    # NEW: Add SeedingConfig and VirtualScrollConfig
+    "SeedingConfig",
+    "VirtualScrollConfig",
+    # NEW: Add AsyncUrlSeeder
+    "AsyncUrlSeeder",
+    # Adaptive Crawler
+    "AdaptiveCrawler",
+    "AdaptiveConfig", 
+    "CrawlState",
+    "CrawlStrategy",
+    "StatisticalStrategy",
     "DeepCrawlStrategy",
     "BFSDeepCrawlStrategy",
     "BestFirstCrawlingStrategy",
@@ -105,6 +139,7 @@ __all__ = [
     "CrawlResult",
     "CrawlerHub",
     "CacheMode",
+    "MatchMode",
     "ContentScrapingStrategy",
     "WebScrapingStrategy",
     "LXMLWebScrapingStrategy",
@@ -117,6 +152,7 @@ __all__ = [
     "JsonCssExtractionStrategy",
     "JsonXPathExtractionStrategy",
     "JsonLxmlExtractionStrategy",
+    "RegexExtractionStrategy",
     "ChunkingStrategy",
     "RegexChunking",
     "DefaultMarkdownGenerator",
@@ -129,11 +165,27 @@ __all__ = [
     "SemaphoreDispatcher",
     "RateLimiter",
     "CrawlerMonitor",
+    "LinkPreview",
     "DisplayMode",
     "MarkdownGenerationResult",
     "Crawl4aiDockerClient",
     "ProxyRotationStrategy",
     "RoundRobinProxyStrategy",
+    "ProxyConfig",
+    "start_colab_display_server",
+    "setup_colab_environment",
+    # C4A Script additions
+    "c4a_compile",
+    "c4a_validate", 
+    "c4a_compile_file",
+    "CompilationResult",
+    "ValidationResult",
+    "ErrorDetail",
+    # Browser Adapters
+    "BrowserAdapter",
+    "PlaywrightAdapter", 
+    "UndetectedAdapter",
+    "LinkPreviewConfig"
 ]
 
 
@@ -162,4 +214,4 @@ __all__ = [
 
 # Disable all Pydantic warnings
 warnings.filterwarnings("ignore", module="pydantic")
-# pydantic_warnings.filter_warnings()
+# pydantic_warnings.filter_warnings()

crawl4ai/__version__.py

@@ -1,2 +1,8 @@
-# crawl4ai/_version.py
-__version__ = "0.5.0.post8"
+# crawl4ai/__version__.py
+
+# This is the version that will be used for stable releases
+__version__ = "0.7.3"
+
+# For nightly builds, this gets set during build process
+__nightly_version__ = None
+

crawl4ai/async_database.py

@@ -171,7 +171,10 @@ class AsyncDatabaseManager:
                             f"Code context:\n{error_context['code_context']}"
                         )
                         self.logger.error(
-                            message=create_box_message(error_message, type="error"),
+                            message="{error}",
+                            tag="ERROR",
+                            params={"error": str(error_message)},
+                            boxes=["error"],
                         )
 
                         raise
@@ -189,7 +192,10 @@ class AsyncDatabaseManager:
                 f"Code context:\n{error_context['code_context']}"
             )
             self.logger.error(
-                message=create_box_message(error_message, type="error"),
+                message="{error}",
+                tag="ERROR",
+                params={"error": str(error_message)},
+                boxes=["error"],
             )
             raise
         finally:

crawl4ai/async_dispatcher.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional, List, Tuple
+from typing import Dict, Optional, List, Tuple, Union
 from .async_configs import CrawlerRunConfig
 from .models import (
     CrawlResult,
@@ -22,6 +22,8 @@ from urllib.parse import urlparse
 import random
 from abc import ABC, abstractmethod
 
+from .memory_utils import get_true_memory_usage_percent
+
 
 class RateLimiter:
     def __init__(
@@ -96,11 +98,37 @@ class BaseDispatcher(ABC):
         self.rate_limiter = rate_limiter
         self.monitor = monitor
 
+    def select_config(self, url: str, configs: Union[CrawlerRunConfig, List[CrawlerRunConfig]]) -> Optional[CrawlerRunConfig]:
+        """Select the appropriate config for a given URL.
+        
+        Args:
+            url: The URL to match against
+            configs: Single config or list of configs to choose from
+            
+        Returns:
+            The matching config, or None if no match found
+        """
+        # Single config - return as is
+        if isinstance(configs, CrawlerRunConfig):
+            return configs
+        
+        # Empty list - return None
+        if not configs:
+            return None
+        
+        # Find first matching config
+        for config in configs:
+            if config.is_match(url):
+                return config
+        
+        # No match found - return None to indicate URL should be skipped
+        return None
+
     @abstractmethod
     async def crawl_url(
         self,
         url: str,
-        config: CrawlerRunConfig,
+        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
         task_id: str,
         monitor: Optional[CrawlerMonitor] = None,
     ) -> CrawlerTaskResult:
@@ -111,7 +139,7 @@ class BaseDispatcher(ABC):
         self,
         urls: List[str],
         crawler: AsyncWebCrawler,  # noqa: F821
-        config: CrawlerRunConfig,
+        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
         monitor: Optional[CrawlerMonitor] = None,
     ) -> List[CrawlerTaskResult]:
         pass
@@ -126,6 +154,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
         check_interval: float = 1.0,
         max_session_permit: int = 20,
         fairness_timeout: float = 600.0,  # 10 minutes before prioritizing long-waiting URLs
+        memory_wait_timeout: Optional[float] = 600.0,
         rate_limiter: Optional[RateLimiter] = None,
         monitor: Optional[CrawlerMonitor] = None,
     ):
@@ -136,27 +165,46 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
         self.check_interval = check_interval
         self.max_session_permit = max_session_permit
         self.fairness_timeout = fairness_timeout
+        self.memory_wait_timeout = memory_wait_timeout
         self.result_queue = asyncio.Queue()
         self.task_queue = asyncio.PriorityQueue()  # Priority queue for better management
         self.memory_pressure_mode = False  # Flag to indicate when we're in memory pressure mode
         self.current_memory_percent = 0.0  # Track current memory usage
+        self._high_memory_start_time: Optional[float] = None
         
     async def _memory_monitor_task(self):
         """Background task to continuously monitor memory usage and update state"""
         while True:
-            self.current_memory_percent = psutil.virtual_memory().percent
-            
+            self.current_memory_percent = get_true_memory_usage_percent()
+
             # Enter memory pressure mode if we cross the threshold
-            if not self.memory_pressure_mode and self.current_memory_percent >= self.memory_threshold_percent:
-                self.memory_pressure_mode = True
-                if self.monitor:
-                    self.monitor.update_memory_status("PRESSURE")
-            
+            if self.current_memory_percent >= self.memory_threshold_percent:
+                if not self.memory_pressure_mode:
+                    self.memory_pressure_mode = True
+                    self._high_memory_start_time = time.time()
+                    if self.monitor:
+                        self.monitor.update_memory_status("PRESSURE")
+                else:
+                    if self._high_memory_start_time is None:
+                        self._high_memory_start_time = time.time()
+                    if (
+                        self.memory_wait_timeout is not None
+                        and self._high_memory_start_time is not None
+                        and time.time() - self._high_memory_start_time >= self.memory_wait_timeout
+                    ):
+                        raise MemoryError(
+                            "Memory usage exceeded threshold for"
+                            f" {self.memory_wait_timeout} seconds"
+                        )
+
             # Exit memory pressure mode if we go below recovery threshold
             elif self.memory_pressure_mode and self.current_memory_percent <= self.recovery_threshold_percent:
                 self.memory_pressure_mode = False
+                self._high_memory_start_time = None
                 if self.monitor:
                     self.monitor.update_memory_status("NORMAL")
+            elif self.current_memory_percent < self.memory_threshold_percent:
+                self._high_memory_start_time = None
             
             # In critical mode, we might need to take more drastic action
             if self.current_memory_percent >= self.critical_threshold_percent:
@@ -180,7 +228,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
     async def crawl_url(
         self,
         url: str,
-        config: CrawlerRunConfig,
+        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
         task_id: str,
         retry_count: int = 0,
     ) -> CrawlerTaskResult:
@@ -188,6 +236,37 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
         error_message = ""
         memory_usage = peak_memory = 0.0
         
+        # Select appropriate config for this URL
+        selected_config = self.select_config(url, config)
+        
+        # If no config matches, return failed result
+        if selected_config is None:
+            error_message = f"No matching configuration found for URL: {url}"
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id, 
+                    status=CrawlStatus.FAILED,
+                    error_message=error_message
+                )
+            
+            return CrawlerTaskResult(
+                task_id=task_id,
+                url=url,
+                result=CrawlResult(
+                    url=url, 
+                    html="", 
+                    metadata={"status": "no_config_match"}, 
+                    success=False, 
+                    error_message=error_message
+                ),
+                memory_usage=0,
+                peak_memory=0,
+                start_time=start_time,
+                end_time=time.time(),
+                error_message=error_message,
+                retry_count=retry_count
+            )
+        
         # Get starting memory for accurate measurement
         process = psutil.Process()
         start_memory = process.memory_info().rss / (1024 * 1024)
@@ -237,8 +316,8 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
                     retry_count=retry_count + 1
                 )
             
-            # Execute the crawl
-            result = await self.crawler.arun(url, config=config, session_id=task_id)
+            # Execute the crawl with selected config
+            result = await self.crawler.arun(url, config=selected_config, session_id=task_id)
             
             # Measure memory usage
             end_memory = process.memory_info().rss / (1024 * 1024)
@@ -296,7 +375,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
         self,
         urls: List[str],
         crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
+        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
     ) -> List[CrawlerTaskResult]:
         self.crawler = crawler
         
@@ -307,7 +386,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             self.monitor.start()
             
         results = []
-        
+
         try:
             # Initialize task queue
             for url in urls:
@@ -316,11 +395,18 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
                     self.monitor.add_task(task_id, url)
                 # Add to queue with initial priority 0, retry count 0, and current time
                 await self.task_queue.put((0, (url, task_id, 0, time.time())))
-                
+
             active_tasks = []
-            
+
             # Process until both queues are empty
             while not self.task_queue.empty() or active_tasks:
+                if memory_monitor.done():
+                    exc = memory_monitor.exception()
+                    if exc:
+                        for t in active_tasks:
+                            t.cancel()
+                        raise exc
+
                 # If memory pressure is low, start new tasks
                 if not self.memory_pressure_mode and len(active_tasks) < self.max_session_permit:
                     try:
@@ -443,7 +529,7 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
         self,
         urls: List[str],
         crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
+        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
     ) -> AsyncGenerator[CrawlerTaskResult, None]:
         self.crawler = crawler
         
@@ -465,8 +551,14 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             active_tasks = []
             completed_count = 0
             total_urls = len(urls)
-            
+
             while completed_count < total_urls:
+                if memory_monitor.done():
+                    exc = memory_monitor.exception()
+                    if exc:
+                        for t in active_tasks:
+                            t.cancel()
+                        raise exc
                 # If memory pressure is low, start new tasks
                 if not self.memory_pressure_mode and len(active_tasks) < self.max_session_permit:
                     try:
@@ -539,7 +631,7 @@ class SemaphoreDispatcher(BaseDispatcher):
     async def crawl_url(
         self,
         url: str,
-        config: CrawlerRunConfig,
+        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
         task_id: str,
         semaphore: asyncio.Semaphore = None,
     ) -> CrawlerTaskResult:
@@ -547,6 +639,36 @@ class SemaphoreDispatcher(BaseDispatcher):
         error_message = ""
         memory_usage = peak_memory = 0.0
 
+        # Select appropriate config for this URL
+        selected_config = self.select_config(url, config)
+        
+        # If no config matches, return failed result
+        if selected_config is None:
+            error_message = f"No matching configuration found for URL: {url}"
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id, 
+                    status=CrawlStatus.FAILED,
+                    error_message=error_message
+                )
+            
+            return CrawlerTaskResult(
+                task_id=task_id,
+                url=url,
+                result=CrawlResult(
+                    url=url, 
+                    html="", 
+                    metadata={"status": "no_config_match"}, 
+                    success=False, 
+                    error_message=error_message
+                ),
+                memory_usage=0,
+                peak_memory=0,
+                start_time=start_time,
+                end_time=time.time(),
+                error_message=error_message
+            )
+
         try:
             if self.monitor:
                 self.monitor.update_task(
@@ -559,7 +681,7 @@ class SemaphoreDispatcher(BaseDispatcher):
             async with semaphore:
                 process = psutil.Process()
                 start_memory = process.memory_info().rss / (1024 * 1024)
-                result = await self.crawler.arun(url, config=config, session_id=task_id)
+                result = await self.crawler.arun(url, config=selected_config, session_id=task_id)
                 end_memory = process.memory_info().rss / (1024 * 1024)
 
                 memory_usage = peak_memory = end_memory - start_memory
@@ -621,7 +743,7 @@ class SemaphoreDispatcher(BaseDispatcher):
         self,
         crawler: AsyncWebCrawler,  # noqa: F821
         urls: List[str],
-        config: CrawlerRunConfig,
+        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
     ) -> List[CrawlerTaskResult]:
         self.crawler = crawler
         if self.monitor:

crawl4ai/async_logger.py

@@ -1,18 +1,49 @@
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import Optional, Dict, Any
-from colorama import Fore, Style, init
+from typing import Optional, Dict, Any, List
 import os
 from datetime import datetime
+from urllib.parse import unquote
+from rich.console import Console
+from rich.text import Text
+from .utils import create_box_message
 
 
 class LogLevel(Enum):
+    DEFAULT = 0
     DEBUG = 1
     INFO = 2
     SUCCESS = 3
     WARNING = 4
     ERROR = 5
+    CRITICAL = 6
+    ALERT = 7
+    NOTICE = 8
+    EXCEPTION = 9
+    FATAL = 10
+    
 
+    def __str__(self):
+        return self.name.lower()
+
+class LogColor(str, Enum):
+    """Enum for log colors."""
+
+    DEBUG = "bright_black"
+    INFO = "cyan"
+    SUCCESS = "green"
+    WARNING = "yellow"
+    ERROR = "red"
+    CYAN = "cyan"
+    GREEN = "green"
+    YELLOW = "yellow"
+    MAGENTA = "magenta"
+    DIM_MAGENTA = "dim magenta"
+    RED = "red"
+
+    def __str__(self):
+        """Automatically convert rich color to string."""
+        return self.value
 
 
 class AsyncLoggerBase(ABC):
@@ -37,13 +68,14 @@ class AsyncLoggerBase(ABC):
         pass
 
     @abstractmethod
-    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 100):
         pass
 
     @abstractmethod
-    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 100):
         pass
 
+
 class AsyncLogger(AsyncLoggerBase):
     """
     Asynchronous logger with support for colored console output and file logging.
@@ -61,14 +93,21 @@ class AsyncLogger(AsyncLoggerBase):
         "DEBUG": "⋯",
         "INFO": "ℹ",
         "WARNING": "⚠",
+        "SUCCESS": "✔",
+        "CRITICAL": "‼",
+        "ALERT": "⚡",
+        "NOTICE": "ℹ",
+        "EXCEPTION": "❗",
+        "FATAL": "☠",
+        "DEFAULT": "•",
     }
 
     DEFAULT_COLORS = {
-        LogLevel.DEBUG: Fore.LIGHTBLACK_EX,
-        LogLevel.INFO: Fore.CYAN,
-        LogLevel.SUCCESS: Fore.GREEN,
-        LogLevel.WARNING: Fore.YELLOW,
-        LogLevel.ERROR: Fore.RED,
+        LogLevel.DEBUG: LogColor.DEBUG,
+        LogLevel.INFO: LogColor.INFO,
+        LogLevel.SUCCESS: LogColor.SUCCESS,
+        LogLevel.WARNING: LogColor.WARNING,
+        LogLevel.ERROR: LogColor.ERROR,
     }
 
     def __init__(
@@ -77,7 +116,7 @@ class AsyncLogger(AsyncLoggerBase):
         log_level: LogLevel = LogLevel.DEBUG,
         tag_width: int = 10,
         icons: Optional[Dict[str, str]] = None,
-        colors: Optional[Dict[LogLevel, str]] = None,
+        colors: Optional[Dict[LogLevel, LogColor]] = None,
         verbose: bool = True,
     ):
         """
@@ -91,13 +130,13 @@ class AsyncLogger(AsyncLoggerBase):
             colors: Custom colors for different log levels
             verbose: Whether to output to console
         """
-        init()  # Initialize colorama
         self.log_file = log_file
         self.log_level = log_level
         self.tag_width = tag_width
         self.icons = icons or self.DEFAULT_ICONS
         self.colors = colors or self.DEFAULT_COLORS
         self.verbose = verbose
+        self.console = Console()
 
         # Create log file directory if needed
         if log_file:
@@ -110,20 +149,23 @@ class AsyncLogger(AsyncLoggerBase):
     def _get_icon(self, tag: str) -> str:
         """Get the icon for a tag, defaulting to info icon if not found."""
         return self.icons.get(tag, self.icons["INFO"])
+    
+    def _shorten(self, text, length, placeholder="..."):
+        """Truncate text in the middle if longer than length, or pad if shorter."""
+        if len(text) <= length:
+            return text.ljust(length)  # Pad with spaces to reach desired length
+        half = (length - len(placeholder)) // 2
+        shortened = text[:half] + placeholder + text[-half:]
+        return shortened.ljust(length)  # Also pad shortened text to consistent length
 
     def _write_to_file(self, message: str):
         """Write a message to the log file if configured."""
         if self.log_file:
+            text = Text.from_markup(message)
+            plain_text = text.plain
             timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
             with open(self.log_file, "a", encoding="utf-8") as f:
-                # Strip ANSI color codes for file output
-                clean_message = message.replace(Fore.RESET, "").replace(
-                    Style.RESET_ALL, ""
-                )
-                for color in vars(Fore).values():
-                    if isinstance(color, str):
-                        clean_message = clean_message.replace(color, "")
-                f.write(f"[{timestamp}] {clean_message}\n")
+                f.write(f"[{timestamp}] {plain_text}\n")
 
     def _log(
         self,
@@ -131,8 +173,9 @@ class AsyncLogger(AsyncLoggerBase):
         message: str,
         tag: str,
         params: Optional[Dict[str, Any]] = None,
-        colors: Optional[Dict[str, str]] = None,
-        base_color: Optional[str] = None,
+        colors: Optional[Dict[str, LogColor]] = None,
+        boxes: Optional[List[str]] = None,
+        base_color: Optional[LogColor] = None,
         **kwargs,
     ):
         """
@@ -144,55 +187,44 @@ class AsyncLogger(AsyncLoggerBase):
             tag: Tag for the message
             params: Parameters to format into the message
             colors: Color overrides for specific parameters
+            boxes: Box overrides for specific parameters
             base_color: Base color for the entire message
         """
         if level.value < self.log_level.value:
             return
 
-        # Format the message with parameters if provided
+        # avoid conflict with rich formatting
+        parsed_message = message.replace("[", "[[").replace("]", "]]")
         if params:
-            try:
-                # First format the message with raw parameters
-                formatted_message = message.format(**params)
+            # FIXME: If there are formatting strings in floating point format, 
+            # this may result in colors and boxes not being applied properly.
+            # such as {value:.2f}, the value is 0.23333 format it to 0.23,
+            # but we replace("0.23333", "[color]0.23333[/color]")
+            formatted_message = parsed_message.format(**params)
+            for key, value in params.items():
+                # value_str may discard `[` and `]`, so we need to replace it. 
+                value_str = str(value).replace("[", "[[").replace("]", "]]")
+                # check is need apply color
+                if colors and key in colors:
+                    color_str = f"[{colors[key]}]{value_str}[/{colors[key]}]"
+                    formatted_message = formatted_message.replace(value_str, color_str)
+                    value_str = color_str
 
-                # Then apply colors if specified
-                color_map = {
-                    "green": Fore.GREEN,
-                    "red": Fore.RED,
-                    "yellow": Fore.YELLOW,
-                    "blue": Fore.BLUE,
-                    "cyan": Fore.CYAN,
-                    "magenta": Fore.MAGENTA,
-                    "white": Fore.WHITE,
-                    "black": Fore.BLACK,
-                    "reset": Style.RESET_ALL,
-                }
-                if colors:
-                    for key, color in colors.items():
-                        # Find the formatted value in the message and wrap it with color
-                        if color in color_map:
-                            color = color_map[color]
-                        if key in params:
-                            value_str = str(params[key])
-                            formatted_message = formatted_message.replace(
-                                value_str, f"{color}{value_str}{Style.RESET_ALL}"
-                            )
+                # check is need apply box
+                if boxes and key in boxes:
+                    formatted_message = formatted_message.replace(value_str,
+                        create_box_message(value_str, type=str(level)))
 
-            except KeyError as e:
-                formatted_message = (
-                    f"LOGGING ERROR: Missing parameter {e} in message template"
-                )
-                level = LogLevel.ERROR
         else:
-            formatted_message = message
+            formatted_message = parsed_message
 
         # Construct the full log line
-        color = base_color or self.colors[level]
-        log_line = f"{color}{self._format_tag(tag)} {self._get_icon(tag)} {formatted_message}{Style.RESET_ALL}"
+        color: LogColor = base_color or self.colors[level]
+        log_line = f"[{color}]{self._format_tag(tag)} {self._get_icon(tag)} {formatted_message} [/{color}]"
 
         # Output to console if verbose
         if self.verbose or kwargs.get("force_verbose", False):
-            print(log_line)
+            self.console.print(log_line)
 
         # Write to file if configured
         self._write_to_file(log_line)
@@ -212,6 +244,22 @@ class AsyncLogger(AsyncLoggerBase):
     def warning(self, message: str, tag: str = "WARNING", **kwargs):
         """Log a warning message."""
         self._log(LogLevel.WARNING, message, tag, **kwargs)
+        
+    def critical(self, message: str, tag: str = "CRITICAL", **kwargs):
+        """Log a critical message."""
+        self._log(LogLevel.ERROR, message, tag, **kwargs)
+    def exception(self, message: str, tag: str = "EXCEPTION", **kwargs):
+        """Log an exception message."""
+        self._log(LogLevel.ERROR, message, tag, **kwargs)
+    def fatal(self, message: str, tag: str = "FATAL", **kwargs):
+        """Log a fatal message."""
+        self._log(LogLevel.ERROR, message, tag, **kwargs)
+    def alert(self, message: str, tag: str = "ALERT", **kwargs):
+        """Log an alert message."""
+        self._log(LogLevel.ERROR, message, tag, **kwargs)
+    def notice(self, message: str, tag: str = "NOTICE", **kwargs):
+        """Log a notice message."""
+        self._log(LogLevel.INFO, message, tag, **kwargs)
 
     def error(self, message: str, tag: str = "ERROR", **kwargs):
         """Log an error message."""
@@ -223,7 +271,7 @@ class AsyncLogger(AsyncLoggerBase):
         success: bool,
         timing: float,
         tag: str = "FETCH",
-        url_length: int = 50,
+        url_length: int = 100,
     ):
         """
         Convenience method for logging URL fetch status.
@@ -235,19 +283,20 @@ class AsyncLogger(AsyncLoggerBase):
             tag: Tag for the message
             url_length: Maximum length for URL in log
         """
+        decoded_url = unquote(url)
+        readable_url = self._shorten(decoded_url, url_length)
         self._log(
             level=LogLevel.SUCCESS if success else LogLevel.ERROR,
-            message="{url:.{url_length}}... | Status: {status} | Time: {timing:.2f}s",
+            message="{url} | {status} | ⏱: {timing:.2f}s",
             tag=tag,
             params={
-                "url": url,
-                "url_length": url_length,
-                "status": success,
+                "url": readable_url,
+                "status": "✓" if success else "✗",
                 "timing": timing,
             },
             colors={
-                "status": Fore.GREEN if success else Fore.RED,
-                "timing": Fore.YELLOW,
+                "status": LogColor.SUCCESS if success else LogColor.ERROR,
+                "timing": LogColor.WARNING,
             },
         )
 
@@ -263,11 +312,13 @@ class AsyncLogger(AsyncLoggerBase):
             tag: Tag for the message
             url_length: Maximum length for URL in log
         """
+        decoded_url = unquote(url)
+        readable_url = self._shorten(decoded_url, url_length)
         self._log(
             level=LogLevel.ERROR,
-            message="{url:.{url_length}}... | Error: {error}",
+            message="{url} | Error: {error}",
             tag=tag,
-            params={"url": url, "url_length": url_length, "error": error},
+            params={"url": readable_url, "error": error},
         )
 
 class AsyncFileLogger(AsyncLoggerBase):
@@ -311,13 +362,13 @@ class AsyncFileLogger(AsyncLoggerBase):
         """Log an error message to file."""
         self._write_to_file("ERROR", message, tag)
 
-    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 100):
         """Log URL fetch status to file."""
         status = "SUCCESS" if success else "FAILED"
         message = f"{url[:url_length]}... | Status: {status} | Time: {timing:.2f}s"
         self._write_to_file("URL_STATUS", message, tag)
 
-    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 100):
         """Log error status to file."""
         message = f"{url[:url_length]}... | Error: {error}"
         self._write_to_file("ERROR", message, tag)

crawl4ai/async_webcrawler.py

@@ -2,7 +2,6 @@ from .__version__ import __version__ as crawl4ai_version
 import os
 import sys
 import time
-from colorama import Fore
 from pathlib import Path
 from typing import Optional, List
 import json
@@ -36,17 +35,18 @@ from .markdown_generation_strategy import (
 )
 from .deep_crawling import DeepCrawlDecorator
 from .async_logger import AsyncLogger, AsyncLoggerBase
-from .async_configs import BrowserConfig, CrawlerRunConfig
+from .async_configs import BrowserConfig, CrawlerRunConfig, ProxyConfig, SeedingConfig
 from .async_dispatcher import *  # noqa: F403
 from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter
+from .async_url_seeder import AsyncUrlSeeder
 
 from .utils import (
     sanitize_input_encode,
     InvalidCSSSelectorError,
     fast_format_html,
-    create_box_message,
     get_error_context,
     RobotsParser,
+    preprocess_html_for_schema,
 )
 
 
@@ -111,7 +111,8 @@ class AsyncWebCrawler:
         self,
         crawler_strategy: AsyncCrawlerStrategy = None,
         config: BrowserConfig = None,
-        base_directory: str = str(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())),
+        base_directory: str = str(
+            os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())),
         thread_safe: bool = False,
         logger: AsyncLoggerBase = None,
         **kwargs,
@@ -139,7 +140,8 @@ class AsyncWebCrawler:
         )
 
         # Initialize crawler strategy
-        params = {k: v for k, v in kwargs.items() if k in ["browser_config", "logger"]}
+        params = {k: v for k, v in kwargs.items() if k in [
+            "browser_config", "logger"]}
         self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
             browser_config=browser_config,
             logger=self.logger,
@@ -162,6 +164,8 @@ class AsyncWebCrawler:
         # Decorate arun method with deep crawling capabilities
         self._deep_handler = DeepCrawlDecorator(self)
         self.arun = self._deep_handler(self.arun)
+        
+        self.url_seeder: Optional[AsyncUrlSeeder] = None
 
     async def start(self):
         """
@@ -237,7 +241,8 @@ class AsyncWebCrawler:
 
         config = config or CrawlerRunConfig()
         if not isinstance(url, str) or not url:
-            raise ValueError("Invalid URL, make sure the URL is a non-empty string")
+            raise ValueError(
+                "Invalid URL, make sure the URL is a non-empty string")
 
         async with self._lock or self.nullcontext():
             try:
@@ -291,12 +296,12 @@ class AsyncWebCrawler:
 
                 # Update proxy configuration from rotation strategy if available
                 if config and config.proxy_rotation_strategy:
-                    next_proxy = await config.proxy_rotation_strategy.get_next_proxy()
+                    next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_next_proxy()
                     if next_proxy:
                         self.logger.info(
                             message="Switch proxy: {proxy}",
                             tag="PROXY",
-                            params={"proxy": next_proxy.server},
+                            params={"proxy": next_proxy.server}
                         )
                         config.proxy_config = next_proxy
                         # config = config.clone(proxy_config=next_proxy)
@@ -306,7 +311,8 @@ class AsyncWebCrawler:
                     t1 = time.perf_counter()
 
                     if config.user_agent:
-                        self.crawler_strategy.update_user_agent(config.user_agent)
+                        self.crawler_strategy.update_user_agent(
+                            config.user_agent)
 
                     # Check robots.txt if enabled
                     if config and config.check_robots_txt:
@@ -353,10 +359,11 @@ class AsyncWebCrawler:
                         html=html,
                         extracted_content=extracted_content,
                         config=config,  # Pass the config object instead of individual parameters
-                        screenshot=screenshot_data,
+                        screenshot_data=screenshot_data,
                         pdf_data=pdf_data,
                         verbose=config.verbose,
                         is_raw_html=True if url.startswith("raw:") else False,
+                        redirected_url=async_response.redirected_url,
                         **kwargs,
                     )
 
@@ -365,25 +372,21 @@ class AsyncWebCrawler:
                     crawl_result.response_headers = async_response.response_headers
                     crawl_result.downloaded_files = async_response.downloaded_files
                     crawl_result.js_execution_result = js_execution_result
-                    crawl_result.ssl_certificate = (
-                        async_response.ssl_certificate
-                    )  # Add SSL certificate
+                    crawl_result.mhtml = async_response.mhtml_data
+                    crawl_result.ssl_certificate = async_response.ssl_certificate
+                    # Add captured network and console data if available
+                    crawl_result.network_requests = async_response.network_requests
+                    crawl_result.console_messages = async_response.console_messages
 
                     crawl_result.success = bool(html)
-                    crawl_result.session_id = getattr(config, "session_id", None)
+                    crawl_result.session_id = getattr(
+                        config, "session_id", None)
 
-                    self.logger.success(
-                        message="{url:.50}... | Status: {status} | Total: {timing}",
+                    self.logger.url_status(
+                        url=cache_context.display_url,
+                        success=crawl_result.success,
+                        timing=time.perf_counter() - start_time,
                         tag="COMPLETE",
-                        params={
-                            "url": cache_context.display_url,
-                            "status": crawl_result.success,
-                            "timing": f"{time.perf_counter() - start_time:.2f}s",
-                        },
-                        colors={
-                            "status": Fore.GREEN if crawl_result.success else Fore.RED,
-                            "timing": Fore.YELLOW,
-                        },
                     )
 
                     # Update cache if appropriate
@@ -393,19 +396,15 @@ class AsyncWebCrawler:
                     return CrawlResultContainer(crawl_result)
 
                 else:
-                    self.logger.success(
-                        message="{url:.50}... | Status: {status} | Total: {timing}",
-                        tag="COMPLETE",
-                        params={
-                            "url": cache_context.display_url,
-                            "status": True,
-                            "timing": f"{time.perf_counter() - start_time:.2f}s",
-                        },
-                        colors={"status": Fore.GREEN, "timing": Fore.YELLOW},
+                    self.logger.url_status(
+                        url=cache_context.display_url,
+                        success=True,
+                        timing=time.perf_counter() - start_time,
+                        tag="COMPLETE"
                     )
-
                     cached_result.success = bool(html)
-                    cached_result.session_id = getattr(config, "session_id", None)
+                    cached_result.session_id = getattr(
+                        config, "session_id", None)
                     cached_result.redirected_url = cached_result.redirected_url or url
                     return CrawlResultContainer(cached_result)
 
@@ -421,7 +420,7 @@ class AsyncWebCrawler:
 
                 self.logger.error_status(
                     url=url,
-                    error=create_box_message(error_message, type="error"),
+                    error=error_message,
                     tag="ERROR",
                 )
 
@@ -437,7 +436,7 @@ class AsyncWebCrawler:
         html: str,
         extracted_content: str,
         config: CrawlerRunConfig,
-        screenshot: str,
+        screenshot_data: str,
         pdf_data: str,
         verbose: bool,
         **kwargs,
@@ -450,7 +449,7 @@ class AsyncWebCrawler:
             html: Raw HTML content
             extracted_content: Previously extracted content (if any)
             config: Configuration object controlling processing behavior
-            screenshot: Screenshot data (if any)
+            screenshot_data: Screenshot data (if any)
             pdf_data: PDF data (if any)
             verbose: Whether to enable verbose logging
             **kwargs: Additional parameters for backwards compatibility
@@ -472,12 +471,14 @@ class AsyncWebCrawler:
             params = config.__dict__.copy()
             params.pop("url", None)
             # add keys from kwargs to params that doesn't exist in params
-            params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
+            params.update({k: v for k, v in kwargs.items()
+                          if k not in params.keys()})
 
             ################################
             # Scraping Strategy Execution  #
             ################################
-            result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
+            result: ScrapingResult = scraping_strategy.scrap(
+                url, html, **params)
 
             if result is None:
                 raise ValueError(
@@ -493,16 +494,24 @@ class AsyncWebCrawler:
 
         # Extract results - handle both dict and ScrapingResult
         if isinstance(result, dict):
-            cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
+            cleaned_html = sanitize_input_encode(
+                result.get("cleaned_html", ""))
             media = result.get("media", {})
+            tables = media.pop("tables", []) if isinstance(media, dict) else []
             links = result.get("links", {})
             metadata = result.get("metadata", {})
         else:
             cleaned_html = sanitize_input_encode(result.cleaned_html)
-            media = result.media.model_dump()
-            links = result.links.model_dump()
+            # media = result.media.model_dump()
+            # tables = media.pop("tables", [])
+            # links = result.links.model_dump()
+            media = result.media.model_dump() if hasattr(result.media, 'model_dump') else result.media
+            tables = media.pop("tables", []) if isinstance(media, dict) else []
+            links = result.links.model_dump() if hasattr(result.links, 'model_dump') else result.links
             metadata = result.metadata
 
+        fit_html = preprocess_html_for_schema(html_content=html, text_threshold= 500, max_size= 300_000)
+
         ################################
         # Generate Markdown            #
         ################################
@@ -510,27 +519,65 @@ class AsyncWebCrawler:
             config.markdown_generator or DefaultMarkdownGenerator()
         )
 
+        # --- SELECT HTML SOURCE BASED ON CONTENT_SOURCE ---
+        # Get the desired source from the generator config, default to 'cleaned_html'
+        selected_html_source = getattr(markdown_generator, 'content_source', 'cleaned_html')
+
+        # Define the source selection logic using dict dispatch
+        html_source_selector = {
+            "raw_html": lambda: html,  # The original raw HTML
+            "cleaned_html": lambda: cleaned_html,  # The HTML after scraping strategy
+            "fit_html": lambda: fit_html,  # The HTML after preprocessing for schema
+        }
+
+        markdown_input_html = cleaned_html  # Default to cleaned_html
+
+        try:
+            # Get the appropriate lambda function, default to returning cleaned_html if key not found
+            source_lambda = html_source_selector.get(selected_html_source, lambda: cleaned_html)
+            # Execute the lambda to get the selected HTML
+            markdown_input_html = source_lambda()
+
+            # Log which source is being used (optional, but helpful for debugging)
+            # if self.logger and verbose:
+            #     actual_source_used = selected_html_source if selected_html_source in html_source_selector else 'cleaned_html (default)'
+            #     self.logger.debug(f"Using '{actual_source_used}' as source for Markdown generation for {url}", tag="MARKDOWN_SRC")
+
+        except Exception as e:
+            # Handle potential errors, especially from preprocess_html_for_schema
+            if self.logger:
+                self.logger.warning(
+                    f"Error getting/processing '{selected_html_source}' for markdown source: {e}. Falling back to cleaned_html.",
+                    tag="MARKDOWN_SRC"
+                )
+            # Ensure markdown_input_html is still the default cleaned_html in case of error
+            markdown_input_html = cleaned_html
+        # --- END: HTML SOURCE SELECTION ---
+
         # Uncomment if by default we want to use PruningContentFilter
         # if not config.content_filter and not markdown_generator.content_filter:
         #     markdown_generator.content_filter = PruningContentFilter()
 
         markdown_result: MarkdownGenerationResult = (
             markdown_generator.generate_markdown(
-                cleaned_html=cleaned_html,
-                base_url=url,
+                input_html=markdown_input_html,
+                base_url=params.get("redirected_url", url)
                 # html2text_options=kwargs.get('html2text', {})
             )
         )
 
         # Log processing completion
-        self.logger.info(
-            message="{url:.50}... | Time: {timing}s",
-            tag="SCRAPE",
-            params={
-                "url": _url,
-                "timing": int((time.perf_counter() - t1) * 1000) / 1000,
-            },
+        self.logger.url_status(
+            url=_url,
+            success=True,
+            timing=int((time.perf_counter() - t1) * 1000) / 1000,
+            tag="SCRAPE"
         )
+        # self.logger.info(
+        #     message="{url:.50}... | Time: {timing}s",
+        #     tag="SCRAPE",
+        #     params={"url": _url, "timing": int((time.perf_counter() - t1) * 1000) / 1000},
+        # )
 
         ################################
         # Structured Content Extraction           #
@@ -544,16 +591,19 @@ class AsyncWebCrawler:
             # Choose content based on input_format
             content_format = config.extraction_strategy.input_format
             if content_format == "fit_markdown" and not markdown_result.fit_markdown:
-                self.logger.warning(
-                    message="Fit markdown requested but not available. Falling back to raw markdown.",
-                    tag="EXTRACT",
-                    params={"url": _url},
-                )
+
+                self.logger.url_status(
+                        url=_url,
+                        success=bool(html),
+                        timing=time.perf_counter() - t1,
+                        tag="EXTRACT",
+                    )
                 content_format = "markdown"
 
             content = {
                 "markdown": markdown_result.raw_markdown,
                 "html": html,
+                "fit_html": fit_html,
                 "cleaned_html": cleaned_html,
                 "fit_markdown": markdown_result.fit_markdown,
             }.get(content_format, markdown_result.raw_markdown)
@@ -561,7 +611,7 @@ class AsyncWebCrawler:
             # Use IdentityChunking for HTML input, otherwise use provided chunking strategy
             chunking = (
                 IdentityChunking()
-                if content_format in ["html", "cleaned_html"]
+                if content_format in ["html", "cleaned_html", "fit_html"]
                 else config.chunking_strategy
             )
             sections = chunking.chunk(content)
@@ -571,15 +621,12 @@ class AsyncWebCrawler:
             )
 
             # Log extraction completion
-            self.logger.info(
-                message="Completed for {url:.50}... | Time: {timing}s",
-                tag="EXTRACT",
-                params={"url": _url, "timing": time.perf_counter() - t1},
-            )
-
-        # Handle screenshot and PDF data
-        screenshot_data = None if not screenshot else screenshot
-        pdf_data = None if not pdf_data else pdf_data
+            self.logger.url_status(
+                        url=_url,
+                        success=bool(html),
+                        timing=time.perf_counter() - t1,
+                        tag="EXTRACT",
+                    )
 
         # Apply HTML formatting if requested
         if config.prettiify:
@@ -589,9 +636,11 @@ class AsyncWebCrawler:
         return CrawlResult(
             url=url,
             html=html,
+            fit_html=fit_html,
             cleaned_html=cleaned_html,
             markdown=markdown_result,
             media=media,
+            tables=tables,                       # NEW
             links=links,
             metadata=metadata,
             screenshot=screenshot_data,
@@ -604,7 +653,7 @@ class AsyncWebCrawler:
     async def arun_many(
         self,
         urls: List[str],
-        config: Optional[CrawlerRunConfig] = None,
+        config: Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]] = None,
         dispatcher: Optional[BaseDispatcher] = None,
         # Legacy parameters maintained for backwards compatibility
         # word_count_threshold=MIN_WORD_THRESHOLD,
@@ -625,7 +674,9 @@ class AsyncWebCrawler:
 
         Args:
         urls: List of URLs to crawl
-        config: Configuration object controlling crawl behavior for all URLs
+        config: Configuration object(s) controlling crawl behavior. Can be:
+            - Single CrawlerRunConfig: Used for all URLs
+            - List[CrawlerRunConfig]: Configs with url_matcher for URL-specific settings
         dispatcher: The dispatcher strategy instance to use. Defaults to MemoryAdaptiveDispatcher
         [other parameters maintained for backwards compatibility]
 
@@ -690,7 +741,11 @@ class AsyncWebCrawler:
                 or task_result.result
             )
 
-        stream = config.stream
+        # Handle stream setting - use first config's stream setting if config is a list
+        if isinstance(config, list):
+            stream = config[0].stream if config else False
+        else:
+            stream = config.stream
 
         if stream:
 
@@ -704,3 +759,94 @@ class AsyncWebCrawler:
         else:
             _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config)
             return [transform_result(res) for res in _results]
+
+    async def aseed_urls(
+        self,
+        domain_or_domains: Union[str, List[str]],
+        config: Optional[SeedingConfig] = None,
+        **kwargs
+    ) -> Union[List[str], Dict[str, List[Union[str, Dict[str, Any]]]]]:
+        """
+        Discovers, filters, and optionally validates URLs for a given domain(s)
+        using sitemaps and Common Crawl archives.
+
+        Args:
+            domain_or_domains: A single domain string (e.g., "iana.org") or a list of domains.
+            config: A SeedingConfig object to control the seeding process.
+                    Parameters passed directly via kwargs will override those in 'config'.
+            **kwargs: Additional parameters (e.g., `source`, `live_check`, `extract_head`,
+                      `pattern`, `concurrency`, `hits_per_sec`, `force_refresh`, `verbose`)
+                      that will be used to construct or update the SeedingConfig.
+
+        Returns:
+            If `extract_head` is False:
+                - For a single domain: `List[str]` of discovered URLs.
+                - For multiple domains: `Dict[str, List[str]]` mapping each domain to its URLs.
+            If `extract_head` is True:
+                - For a single domain: `List[Dict[str, Any]]` where each dict contains 'url'
+                  and 'head_data' (parsed <head> metadata).
+                - For multiple domains: `Dict[str, List[Dict[str, Any]]]` mapping each domain
+                  to a list of URL data dictionaries.
+
+        Raises:
+            ValueError: If `domain_or_domains` is not a string or a list of strings.
+            Exception: Any underlying exceptions from AsyncUrlSeeder or network operations.
+
+        Example:
+            >>> # Discover URLs from sitemap with live check for 'example.com'
+            >>> result = await crawler.aseed_urls("example.com", source="sitemap", live_check=True, hits_per_sec=10)
+
+            >>> # Discover URLs from Common Crawl, extract head data for 'example.com' and 'python.org'
+            >>> multi_domain_result = await crawler.aseed_urls(
+            >>>     ["example.com", "python.org"],
+            >>>     source="cc", extract_head=True, concurrency=200, hits_per_sec=50
+            >>> )
+        """
+        # Initialize AsyncUrlSeeder here if it hasn't been already
+        if not self.url_seeder:
+            # Pass the crawler's base_directory for seeder's cache management
+            # Pass the crawler's logger for consistent logging
+            self.url_seeder = AsyncUrlSeeder(
+                base_directory=self.crawl4ai_folder,
+                logger=self.logger
+            )                    
+
+        # Merge config object with direct kwargs, giving kwargs precedence
+        seeding_config = config.clone(**kwargs) if config else SeedingConfig.from_kwargs(kwargs)
+        
+        # Ensure base_directory is set for the seeder's cache
+        seeding_config.base_directory = seeding_config.base_directory or self.crawl4ai_folder        
+        # Ensure the seeder uses the crawler's logger (if not already set)
+        if not self.url_seeder.logger:
+            self.url_seeder.logger = self.logger
+
+        # Pass verbose setting if explicitly provided in SeedingConfig or kwargs
+        if seeding_config.verbose is not None:
+            self.url_seeder.logger.verbose = seeding_config.verbose
+        else: # Default to crawler's verbose setting
+            self.url_seeder.logger.verbose = self.logger.verbose
+
+
+        if isinstance(domain_or_domains, str):
+            self.logger.info(
+                message="Starting URL seeding for domain: {domain}",
+                tag="SEED",
+                params={"domain": domain_or_domains}
+            )
+            return await self.url_seeder.urls(
+                domain_or_domains,
+                seeding_config
+            )
+        elif isinstance(domain_or_domains, (list, tuple)):
+            self.logger.info(
+                message="Starting URL seeding for {count} domains",
+                tag="SEED",
+                params={"count": len(domain_or_domains)}
+            )
+            # AsyncUrlSeeder.many_urls directly accepts a list of domains and individual params.
+            return await self.url_seeder.many_urls(
+                domain_or_domains,
+                seeding_config
+            )
+        else:
+            raise ValueError("`domain_or_domains` must be a string or a list of strings.")

crawl4ai/browser/__init__.py

@@ -1,23 +0,0 @@
-"""Browser management module for Crawl4AI.
-
-This module provides browser management capabilities using different strategies
-for browser creation and interaction.
-"""
-
-from .manager import BrowserManager
-from .profiles import BrowserProfileManager
-from .models import DockerConfig
-from .docker_registry import DockerRegistry
-from .docker_utils import DockerUtils
-from .browser_hub import BrowserHub
-from .strategies import (
-    BaseBrowserStrategy,
-    PlaywrightBrowserStrategy,
-    CDPBrowserStrategy,
-    BuiltinBrowserStrategy,
-    DockerBrowserStrategy
-)
-
-__all__ = ['BrowserManager', 'BrowserProfileManager', 'DockerConfig', 'DockerRegistry', 'DockerUtils', 'BaseBrowserStrategy',
-           'PlaywrightBrowserStrategy', 'CDPBrowserStrategy', 'BuiltinBrowserStrategy',
-           'DockerBrowserStrategy', 'BrowserHub']

crawl4ai/browser/browser_hub.py

@@ -1,184 +0,0 @@
-# browser_hub_manager.py
-import hashlib
-import json
-import asyncio
-from typing import Dict, Optional, List, Tuple
-from .manager import BrowserManager, UnavailableBehavior
-from ..async_configs import BrowserConfig, CrawlerRunConfig
-from ..async_logger import AsyncLogger
-
-class BrowserHub:
-    """
-    Manages Browser-Hub instances for sharing across multiple pipelines.
-    
-    This class provides centralized management for browser resources, allowing
-    multiple pipelines to share browser instances efficiently, connect to
-    existing browser hubs, or create new ones with custom configurations.
-    """
-    _instances: Dict[str, BrowserManager] = {}
-    _lock = asyncio.Lock()
-    
-    @classmethod
-    async def get_browser_manager(
-        cls, 
-        config: Optional[BrowserConfig] = None,
-        hub_id: Optional[str] = None,
-        connection_info: Optional[str] = None,
-        logger: Optional[AsyncLogger] = None,
-        max_browsers_per_config: int = 10,
-        max_pages_per_browser: int = 5,
-        initial_pool_size: int = 1,
-        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
-    ) -> BrowserManager:
-        """
-        Get an existing BrowserManager or create a new one based on parameters.
-        
-        Args:
-            config: Browser configuration for new hub
-            hub_id: Identifier for the hub instance
-            connection_info: Connection string for existing hub
-            logger: Logger for recording events and errors
-            max_browsers_per_config: Maximum browsers per configuration
-            max_pages_per_browser: Maximum pages per browser
-            initial_pool_size: Initial number of browsers to create
-            page_configs: Optional configurations for pre-warming pages
-            
-        Returns:
-            BrowserManager: The requested browser manager instance
-        """
-        async with cls._lock:
-            # Scenario 3: Use existing hub via connection info
-            if connection_info:
-                instance_key = f"connection:{connection_info}"
-                if instance_key not in cls._instances:
-                    cls._instances[instance_key] = await cls._connect_to_browser_hub(
-                        connection_info, logger
-                    )
-                return cls._instances[instance_key]
-                
-            # Scenario 2: Custom configured hub
-            if config:
-                config_hash = cls._hash_config(config)
-                instance_key = hub_id or f"config:{config_hash}"
-                if instance_key not in cls._instances:
-                    cls._instances[instance_key] = await cls._create_browser_manager(
-                        config, 
-                        logger,
-                        max_browsers_per_config,
-                        max_pages_per_browser,
-                        initial_pool_size,
-                        page_configs
-                    )
-                return cls._instances[instance_key]
-            
-            # Scenario 1: Default hub
-            instance_key = "default"
-            if instance_key not in cls._instances:
-                cls._instances[instance_key] = await cls._create_default_browser_hub(
-                    logger,
-                    max_browsers_per_config,
-                    max_pages_per_browser,
-                    initial_pool_size
-                )
-            return cls._instances[instance_key]
-    
-    @classmethod
-    async def _create_browser_manager(
-        cls, 
-        config: BrowserConfig,
-        logger: Optional[AsyncLogger],
-        max_browsers_per_config: int,
-        max_pages_per_browser: int,
-        initial_pool_size: int,
-        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
-    ) -> BrowserManager:
-        """Create a new browser hub with the specified configuration."""
-        manager = BrowserManager(
-            browser_config=config,
-            logger=logger,
-            unavailable_behavior=UnavailableBehavior.ON_DEMAND,
-            max_browsers_per_config=max_browsers_per_config,
-            max_pages_per_browser=max_pages_per_browser,
-        )
-        
-        # Initialize the pool
-        await manager.initialize_pool(
-            browser_configs=[config] if config else None,
-            browsers_per_config=initial_pool_size,
-            page_configs=page_configs
-        )
-        
-        return manager
-    
-    @classmethod
-    async def _create_default_browser_hub(
-        cls,
-        logger: Optional[AsyncLogger],
-        max_browsers_per_config: int,
-        max_pages_per_browser: int,
-        initial_pool_size: int
-    ) -> BrowserManager:
-        """Create a default browser hub with standard settings."""
-        config = BrowserConfig(headless=True)
-        return await cls._create_browser_manager(
-            config, 
-            logger, 
-            max_browsers_per_config, 
-            max_pages_per_browser, 
-            initial_pool_size,
-            None
-        )
-    
-    @classmethod
-    async def _connect_to_browser_hub(
-        cls, 
-        connection_info: str,
-        logger: Optional[AsyncLogger]
-    ) -> BrowserManager:
-        """
-        Connect to an existing browser hub.
-        
-        Note: This is a placeholder for future remote connection functionality.
-        Currently creates a local instance.
-        """
-        if logger:
-            logger.info(
-                message="Remote browser hub connections not yet implemented. Creating local instance.",
-                tag="BROWSER_HUB"
-            )
-        # For now, create a default local instance
-        return await cls._create_default_browser_hub(
-            logger, 
-            max_browsers_per_config=10, 
-            max_pages_per_browser=5, 
-            initial_pool_size=1
-        )
-    
-    @classmethod
-    def _hash_config(cls, config: BrowserConfig) -> str:
-        """Create a hash of the browser configuration for identification."""
-        # Convert config to dictionary, excluding any callable objects
-        config_dict = config.__dict__.copy()
-        for key in list(config_dict.keys()):
-            if callable(config_dict[key]):
-                del config_dict[key]
-        
-        # Convert to canonical JSON string
-        config_json = json.dumps(config_dict, sort_keys=True, default=str)
-        
-        # Hash the JSON
-        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
-        return config_hash
-    
-    @classmethod
-    async def shutdown_all(cls):
-        """Close all browser hub instances and clear the registry."""
-        async with cls._lock:
-            shutdown_tasks = []
-            for hub in cls._instances.values():
-                shutdown_tasks.append(hub.close())
-            
-            if shutdown_tasks:
-                await asyncio.gather(*shutdown_tasks)
-            
-            cls._instances.clear()

crawl4ai/browser/docker/alpine/connect.Dockerfile

@@ -1,34 +0,0 @@
-# ---------- Dockerfile ----------
-    FROM alpine:latest
-
-    # Combine everything in one RUN to keep layers minimal.
-    RUN apk update && apk upgrade && \
-        apk add --no-cache \
-            chromium \
-            nss \
-            freetype \
-            harfbuzz \
-            ca-certificates \
-            ttf-freefont \
-            socat \
-            curl && \
-        addgroup -S chromium && adduser -S chromium -G chromium && \
-        mkdir -p /data && chown chromium:chromium /data && \
-        rm -rf /var/cache/apk/*
-    
-    # Copy start script, then chown/chmod in one step
-    COPY start.sh /home/chromium/start.sh
-    RUN chown chromium:chromium /home/chromium/start.sh && \
-        chmod +x /home/chromium/start.sh
-    
-    USER chromium
-    WORKDIR /home/chromium
-    
-    # Expose port used by socat (mapping 9222→9223 or whichever you prefer)
-    EXPOSE 9223
-    
-    # Simple healthcheck: is the remote debug endpoint responding?
-    HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -f http://localhost:9222/json/version || exit 1
-    
-    CMD ["./start.sh"]
-    

crawl4ai/browser/docker/alpine/launch.Dockerfile

@@ -1,27 +0,0 @@
-# ---------- Dockerfile (Idle Version) ----------
-    FROM alpine:latest
-
-    # Install only Chromium and its dependencies in a single layer
-    RUN apk update && apk upgrade && \
-        apk add --no-cache \
-            chromium \
-            nss \
-            freetype \
-            harfbuzz \
-            ca-certificates \
-            ttf-freefont \
-            socat \
-            curl && \
-        addgroup -S chromium && adduser -S chromium -G chromium && \
-        mkdir -p /data && chown chromium:chromium /data && \
-        rm -rf /var/cache/apk/*
-    
-    ENV PATH="/usr/bin:/bin:/usr/sbin:/sbin"
-
-    # Switch to a non-root user for security
-    USER chromium
-    WORKDIR /home/chromium
-    
-    # Idle: container does nothing except stay alive
-    CMD ["tail", "-f", "/dev/null"]
-    

crawl4ai/browser/docker/debian/connect.Dockerfile

@@ -1,23 +0,0 @@
-# Use Debian 12 (Bookworm) slim for a small, stable base image
-FROM debian:bookworm-slim
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Install Chromium, socat, and basic fonts
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    chromium \
-    wget \
-    curl \
-    socat \
-    fonts-freefont-ttf \
-    fonts-noto-color-emoji && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
-
-# Copy start.sh and make it executable
-COPY start.sh /start.sh
-RUN chmod +x /start.sh
-
-# Expose socat port (use host mapping, e.g. -p 9225:9223)
-EXPOSE 9223
-
-ENTRYPOINT ["/start.sh"]

crawl4ai/browser/docker_registry.py

@@ -1,264 +0,0 @@
-"""Docker registry module for Crawl4AI.
-
-This module provides a registry system for tracking and reusing Docker containers
-across browser sessions, improving performance and resource utilization.
-"""
-
-import os
-import json
-import time
-from typing import Dict, Optional
-
-from ..utils import get_home_folder
-
-
-class DockerRegistry:
-    """Manages a registry of Docker containers used for browser automation.
-    
-    This registry tracks containers by configuration hash, allowing reuse of appropriately
-    configured containers instead of creating new ones for each session.
-    
-    Attributes:
-        registry_file (str): Path to the registry file
-        containers (dict): Dictionary of container information
-        port_map (dict): Map of host ports to container IDs
-        last_port (int): Last port assigned
-    """
-    
-    def __init__(self, registry_file: Optional[str] = None):
-        """Initialize the registry with an optional path to the registry file.
-        
-        Args:
-            registry_file: Path to the registry file. If None, uses default path.
-        """
-        # Use the same file path as BuiltinBrowserStrategy by default
-        self.registry_file = registry_file or os.path.join(get_home_folder(), "builtin-browser", "browser_config.json")
-        self.containers = {}  # Still maintain this for backward compatibility
-        self.port_map = {}    # Will be populated from the shared file
-        self.last_port = 9222
-        self.load()
-    
-    def load(self):
-        """Load container registry from file."""
-        if os.path.exists(self.registry_file):
-            try:
-                with open(self.registry_file, 'r') as f:
-                    registry_data = json.load(f)
-                    
-                    # Initialize port_map if not present
-                    if "port_map" not in registry_data:
-                        registry_data["port_map"] = {}
-                    
-                    self.port_map = registry_data.get("port_map", {})
-                    
-                    # Extract container information from port_map entries of type "docker"
-                    self.containers = {}
-                    for port_str, browser_info in self.port_map.items():
-                        if browser_info.get("browser_type") == "docker" and "container_id" in browser_info:
-                            container_id = browser_info["container_id"]
-                            self.containers[container_id] = {
-                                "host_port": int(port_str),
-                                "config_hash": browser_info.get("config_hash", ""),
-                                "created_at": browser_info.get("created_at", time.time())
-                            }
-                    
-                    # Get last port if available
-                    if "last_port" in registry_data:
-                        self.last_port = registry_data["last_port"]
-                    else:
-                        # Find highest port in port_map
-                        ports = [int(p) for p in self.port_map.keys() if p.isdigit()]
-                        self.last_port = max(ports + [9222])
-                    
-            except Exception as e:
-                # Reset to defaults on error
-                print(f"Error loading registry: {e}")
-                self.containers = {}
-                self.port_map = {}
-                self.last_port = 9222
-        else:
-            # Initialize with defaults if file doesn't exist
-            self.containers = {}
-            self.port_map = {}
-            self.last_port = 9222
-    
-    def save(self):
-        """Save container registry to file."""
-        # First load the current file to avoid overwriting other browser types
-        current_data = {"port_map": {}, "last_port": self.last_port}
-        if os.path.exists(self.registry_file):
-            try:
-                with open(self.registry_file, 'r') as f:
-                    current_data = json.load(f)
-            except Exception:
-                pass
-        
-        # Create a new port_map dictionary
-        updated_port_map = {}
-        
-        # First, copy all non-docker entries from the existing port_map
-        for port_str, browser_info in current_data.get("port_map", {}).items():
-            if browser_info.get("browser_type") != "docker":
-                updated_port_map[port_str] = browser_info
-        
-        # Then add all current docker container entries
-        for container_id, container_info in self.containers.items():
-            port_str = str(container_info["host_port"])
-            updated_port_map[port_str] = {
-                "browser_type": "docker",
-                "container_id": container_id,
-                "cdp_url": f"http://localhost:{port_str}",
-                "config_hash": container_info["config_hash"],
-                "created_at": container_info["created_at"]
-            }
-        
-        # Replace the port_map with our updated version
-        current_data["port_map"] = updated_port_map
-        
-        # Update last_port
-        current_data["last_port"] = self.last_port
-        
-        # Ensure directory exists
-        os.makedirs(os.path.dirname(self.registry_file), exist_ok=True)
-        
-        # Save the updated data
-        with open(self.registry_file, 'w') as f:
-            json.dump(current_data, f, indent=2)
-    
-    def register_container(self, container_id: str, host_port: int, config_hash: str, cdp_json_config: Optional[str] = None):
-        """Register a container with its configuration hash and port mapping.
-        
-        Args:
-            container_id: Docker container ID
-            host_port: Host port mapped to container
-            config_hash: Hash of configuration used to create container
-            cdp_json_config: CDP JSON configuration if available
-        """
-        self.containers[container_id] = {
-            "host_port": host_port,
-            "config_hash": config_hash,
-            "created_at": time.time()
-        }
-        
-        # Update port_map to maintain compatibility with BuiltinBrowserStrategy
-        port_str = str(host_port)
-        self.port_map[port_str] = {
-            "browser_type": "docker",
-            "container_id": container_id,
-            "cdp_url": f"http://localhost:{port_str}",
-            "config_hash": config_hash,
-            "created_at": time.time()
-        }
-        
-        if cdp_json_config:
-            self.port_map[port_str]["cdp_json_config"] = cdp_json_config
-        
-        self.save()
-    
-    def unregister_container(self, container_id: str):
-        """Unregister a container.
-        
-        Args:
-            container_id: Docker container ID to unregister
-        """
-        if container_id in self.containers:
-            host_port = self.containers[container_id]["host_port"]
-            port_str = str(host_port)
-            
-            # Remove from port_map
-            if port_str in self.port_map:
-                del self.port_map[port_str]
-                
-            # Remove from containers
-            del self.containers[container_id]
-            
-            self.save()
-    
-    async def find_container_by_config(self, config_hash: str, docker_utils) -> Optional[str]:
-        """Find a container that matches the given configuration hash.
-        
-        Args:
-            config_hash: Hash of configuration to match
-            docker_utils: DockerUtils instance to check running containers
-            
-        Returns:
-            Container ID if found, None otherwise
-        """
-        # Search through port_map for entries with matching config_hash
-        for port_str, browser_info in self.port_map.items():
-            if (browser_info.get("browser_type") == "docker" and 
-                browser_info.get("config_hash") == config_hash and 
-                "container_id" in browser_info):
-                
-                container_id = browser_info["container_id"]
-                if await docker_utils.is_container_running(container_id):
-                    return container_id
-        
-        return None
-    
-    def get_container_host_port(self, container_id: str) -> Optional[int]:
-        """Get the host port mapped to the container.
-        
-        Args:
-            container_id: Docker container ID
-            
-        Returns:
-            Host port if container is registered, None otherwise
-        """
-        if container_id in self.containers:
-            return self.containers[container_id]["host_port"]
-        return None
-    
-    def get_next_available_port(self, docker_utils) -> int:
-        """Get the next available host port for Docker mapping.
-        
-        Args:
-            docker_utils: DockerUtils instance to check port availability
-            
-        Returns:
-            Available port number
-        """
-        # Start from last port + 1
-        port = self.last_port + 1
-        
-        # Check if port is in use (either in our registry or system-wide)
-        while str(port) in self.port_map or docker_utils.is_port_in_use(port):
-            port += 1
-        
-        # Update last port
-        self.last_port = port
-        self.save()
-        
-        return port
-    
-    def get_container_config_hash(self, container_id: str) -> Optional[str]:
-        """Get the configuration hash for a container.
-        
-        Args:
-            container_id: Docker container ID
-            
-        Returns:
-            Configuration hash if container is registered, None otherwise
-        """
-        if container_id in self.containers:
-            return self.containers[container_id]["config_hash"]
-        return None
-    
-    def cleanup_stale_containers(self, docker_utils):
-        """Clean up containers that are no longer running.
-        
-        Args:
-            docker_utils: DockerUtils instance to check container status
-        """
-        to_remove = []
-        
-        # Find containers that are no longer running
-        for port_str, browser_info in self.port_map.items():
-            if browser_info.get("browser_type") == "docker" and "container_id" in browser_info:
-                container_id = browser_info["container_id"]
-                if not docker_utils.is_container_running(container_id):
-                    to_remove.append(container_id)
-        
-        # Remove stale containers
-        for container_id in to_remove:
-            self.unregister_container(container_id)

crawl4ai/browser/docker_utils.py

@@ -1,661 +0,0 @@
-import os
-import json
-import asyncio
-import hashlib
-import tempfile
-import shutil
-import socket
-import subprocess
-from typing import Dict, List, Optional, Tuple, Union
-
-
-class DockerUtils:
-    """Utility class for Docker operations in browser automation.
-
-    This class provides methods for managing Docker images, containers,
-    and related operations needed for browser automation. It handles
-    image building, container lifecycle, port management, and registry operations.
-
-    Attributes:
-        DOCKER_FOLDER (str): Path to folder containing Docker files
-        DOCKER_CONNECT_FILE (str): Path to Dockerfile for connect mode
-        DOCKER_LAUNCH_FILE (str): Path to Dockerfile for launch mode
-        DOCKER_START_SCRIPT (str): Path to startup script for connect mode
-        DEFAULT_CONNECT_IMAGE (str): Default image name for connect mode
-        DEFAULT_LAUNCH_IMAGE (str): Default image name for launch mode
-        logger: Optional logger instance
-    """
-
-    # File paths for Docker resources
-    DOCKER_FOLDER = os.path.join(os.path.dirname(__file__), "docker")
-    DOCKER_CONNECT_FILE = os.path.join(DOCKER_FOLDER, "connect.Dockerfile")
-    DOCKER_LAUNCH_FILE = os.path.join(DOCKER_FOLDER, "launch.Dockerfile")
-    DOCKER_START_SCRIPT = os.path.join(DOCKER_FOLDER, "start.sh")
-
-    # Default image names
-    DEFAULT_CONNECT_IMAGE = "crawl4ai/browser-connect:latest"
-    DEFAULT_LAUNCH_IMAGE = "crawl4ai/browser-launch:latest"
-
-    def __init__(self, logger=None):
-        """Initialize Docker utilities.
-
-        Args:
-            logger: Optional logger for recording operations
-        """
-        self.logger = logger
-
-    # Image Management Methods
-
-    async def check_image_exists(self, image_name: str) -> bool:
-        """Check if a Docker image exists.
-
-        Args:
-            image_name: Name of the Docker image to check
-
-        Returns:
-            bool: True if the image exists, False otherwise
-        """
-        cmd = ["docker", "image", "inspect", image_name]
-
-        try:
-            process = await asyncio.create_subprocess_exec(
-                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            _, _ = await process.communicate()
-            return process.returncode == 0
-        except Exception as e:
-            if self.logger:
-                self.logger.debug(
-                    f"Error checking if image exists: {str(e)}", tag="DOCKER"
-                )
-            return False
-
-    async def build_docker_image(
-        self,
-        image_name: str,
-        dockerfile_path: str,
-        files_to_copy: Dict[str, str] = None,
-    ) -> bool:
-        """Build a Docker image from a Dockerfile.
-
-        Args:
-            image_name: Name to give the built image
-            dockerfile_path: Path to the Dockerfile
-            files_to_copy: Dict of {dest_name: source_path} for files to copy to build context
-
-        Returns:
-            bool: True if image was built successfully, False otherwise
-        """
-        # Create a temporary build context
-        with tempfile.TemporaryDirectory() as temp_dir:
-            # Copy the Dockerfile
-            shutil.copy(dockerfile_path, os.path.join(temp_dir, "Dockerfile"))
-
-            # Copy any additional files needed
-            if files_to_copy:
-                for dest_name, source_path in files_to_copy.items():
-                    shutil.copy(source_path, os.path.join(temp_dir, dest_name))
-
-            # Build the image
-            cmd = ["docker", "build", "-t", image_name, temp_dir]
-
-            if self.logger:
-                self.logger.debug(
-                    f"Building Docker image with command: {' '.join(cmd)}", tag="DOCKER"
-                )
-
-            process = await asyncio.create_subprocess_exec(
-                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            stdout, stderr = await process.communicate()
-
-            if process.returncode != 0:
-                if self.logger:
-                    self.logger.error(
-                        message="Failed to build Docker image: {error}",
-                        tag="DOCKER",
-                        params={"error": stderr.decode()},
-                    )
-                return False
-
-            if self.logger:
-                self.logger.success(
-                    f"Successfully built Docker image: {image_name}", tag="DOCKER"
-                )
-            return True
-
-    async def ensure_docker_image_exists(
-        self, image_name: str, mode: str = "connect"
-    ) -> str:
-        """Ensure the required Docker image exists, creating it if necessary.
-
-        Args:
-            image_name: Name of the Docker image
-            mode: Either "connect" or "launch" to determine which image to build
-
-        Returns:
-            str: Name of the available Docker image
-
-        Raises:
-            Exception: If image doesn't exist and can't be built
-        """
-        # If image name is not specified, use default based on mode
-        if not image_name:
-            image_name = (
-                self.DEFAULT_CONNECT_IMAGE
-                if mode == "connect"
-                else self.DEFAULT_LAUNCH_IMAGE
-            )
-
-        # Check if the image already exists
-        if await self.check_image_exists(image_name):
-            if self.logger:
-                self.logger.debug(
-                    f"Docker image {image_name} already exists", tag="DOCKER"
-                )
-            return image_name
-
-        # If we're using a custom image that doesn't exist, warn and fail
-        if (
-            image_name != self.DEFAULT_CONNECT_IMAGE
-            and image_name != self.DEFAULT_LAUNCH_IMAGE
-        ):
-            if self.logger:
-                self.logger.warning(
-                    f"Custom Docker image {image_name} not found and cannot be automatically created",
-                    tag="DOCKER",
-                )
-            raise Exception(f"Docker image {image_name} not found")
-
-        # Build the appropriate default image
-        if self.logger:
-            self.logger.info(
-                f"Docker image {image_name} not found, creating it now...", tag="DOCKER"
-            )
-
-        if mode == "connect":
-            success = await self.build_docker_image(
-                image_name,
-                self.DOCKER_CONNECT_FILE,
-                {"start.sh": self.DOCKER_START_SCRIPT},
-            )
-        else:
-            success = await self.build_docker_image(image_name, self.DOCKER_LAUNCH_FILE)
-
-        if not success:
-            raise Exception(f"Failed to create Docker image {image_name}")
-
-        return image_name
-
-    # Container Management Methods
-
-    async def create_container(
-        self,
-        image_name: str,
-        host_port: int,
-        container_name: Optional[str] = None,
-        volumes: List[str] = None,
-        network: Optional[str] = None,
-        env_vars: Dict[str, str] = None,
-        cpu_limit: float = 1.0,
-        memory_limit: str = "1.5g",
-        extra_args: List[str] = None,
-    ) -> Optional[str]:
-        """Create a new Docker container.
-
-        Args:
-            image_name: Docker image to use
-            host_port: Port on host to map to container port 9223
-            container_name: Optional name for the container
-            volumes: List of volume mappings (e.g., ["host_path:container_path"])
-            network: Optional Docker network to use
-            env_vars: Dictionary of environment variables
-            cpu_limit: CPU limit for the container
-            memory_limit: Memory limit for the container
-            extra_args: Additional docker run arguments
-
-        Returns:
-            str: Container ID if successful, None otherwise
-        """
-        # Prepare container command
-        cmd = [
-            "docker",
-            "run",
-            "--detach",
-        ]
-
-        # Add container name if specified
-        if container_name:
-            cmd.extend(["--name", container_name])
-
-        # Add port mapping
-        cmd.extend(["-p", f"{host_port}:9223"])
-
-        # Add volumes
-        if volumes:
-            for volume in volumes:
-                cmd.extend(["-v", volume])
-
-        # Add network if specified
-        if network:
-            cmd.extend(["--network", network])
-
-        # Add environment variables
-        if env_vars:
-            for key, value in env_vars.items():
-                cmd.extend(["-e", f"{key}={value}"])
-
-        # Add CPU and memory limits
-        if cpu_limit:
-            cmd.extend(["--cpus", str(cpu_limit)])
-        if memory_limit:
-            cmd.extend(["--memory", memory_limit])
-            cmd.extend(["--memory-swap", memory_limit])
-        if self.logger:
-            self.logger.debug(
-                f"Setting CPU limit: {cpu_limit}, Memory limit: {memory_limit}",
-                tag="DOCKER",
-            )
-
-        # Add extra args
-        if extra_args:
-            cmd.extend(extra_args)
-
-        # Add image
-        cmd.append(image_name)
-
-        if self.logger:
-            self.logger.debug(
-                f"Creating Docker container with command: {' '.join(cmd)}", tag="DOCKER"
-            )
-
-        # Run docker command
-        try:
-            process = await asyncio.create_subprocess_exec(
-                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            stdout, stderr = await process.communicate()
-
-            if process.returncode != 0:
-                if self.logger:
-                    self.logger.error(
-                        message="Failed to create Docker container: {error}",
-                        tag="DOCKER",
-                        params={"error": stderr.decode()},
-                    )
-                return None
-
-            # Get container ID
-            container_id = stdout.decode().strip()
-
-            if self.logger:
-                self.logger.success(
-                    f"Created Docker container: {container_id[:12]}", tag="DOCKER"
-                )
-
-            return container_id
-
-        except Exception as e:
-            if self.logger:
-                self.logger.error(
-                    message="Error creating Docker container: {error}",
-                    tag="DOCKER",
-                    params={"error": str(e)},
-                )
-            return None
-
-    async def is_container_running(self, container_id: str) -> bool:
-        """Check if a container is running.
-
-        Args:
-            container_id: ID of the container to check
-
-        Returns:
-            bool: True if the container is running, False otherwise
-        """
-        cmd = ["docker", "inspect", "--format", "{{.State.Running}}", container_id]
-
-        try:
-            process = await asyncio.create_subprocess_exec(
-                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            stdout, _ = await process.communicate()
-
-            return process.returncode == 0 and stdout.decode().strip() == "true"
-        except Exception as e:
-            if self.logger:
-                self.logger.debug(
-                    f"Error checking if container is running: {str(e)}", tag="DOCKER"
-                )
-            return False
-
-    async def wait_for_container_ready(
-        self, container_id: str, timeout: int = 30
-    ) -> bool:
-        """Wait for the container to be in running state.
-
-        Args:
-            container_id: ID of the container to wait for
-            timeout: Maximum time to wait in seconds
-
-        Returns:
-            bool: True if container is ready, False if timeout occurred
-        """
-        for _ in range(timeout):
-            if await self.is_container_running(container_id):
-                return True
-            await asyncio.sleep(1)
-
-        if self.logger:
-            self.logger.warning(
-                f"Container {container_id[:12]} not ready after {timeout}s timeout",
-                tag="DOCKER",
-            )
-        return False
-
-    async def stop_container(self, container_id: str) -> bool:
-        """Stop a Docker container.
-
-        Args:
-            container_id: ID of the container to stop
-
-        Returns:
-            bool: True if stopped successfully, False otherwise
-        """
-        cmd = ["docker", "stop", container_id]
-
-        try:
-            process = await asyncio.create_subprocess_exec(*cmd)
-            await process.communicate()
-
-            if self.logger:
-                self.logger.debug(
-                    f"Stopped container: {container_id[:12]}", tag="DOCKER"
-                )
-
-            return process.returncode == 0
-        except Exception as e:
-            if self.logger:
-                self.logger.warning(
-                    message="Failed to stop container: {error}",
-                    tag="DOCKER",
-                    params={"error": str(e)},
-                )
-            return False
-
-    async def remove_container(self, container_id: str, force: bool = True) -> bool:
-        """Remove a Docker container.
-
-        Args:
-            container_id: ID of the container to remove
-            force: Whether to force removal
-
-        Returns:
-            bool: True if removed successfully, False otherwise
-        """
-        cmd = ["docker", "rm"]
-        if force:
-            cmd.append("-f")
-        cmd.append(container_id)
-
-        try:
-            process = await asyncio.create_subprocess_exec(*cmd)
-            await process.communicate()
-
-            if self.logger:
-                self.logger.debug(
-                    f"Removed container: {container_id[:12]}", tag="DOCKER"
-                )
-
-            return process.returncode == 0
-        except Exception as e:
-            if self.logger:
-                self.logger.warning(
-                    message="Failed to remove container: {error}",
-                    tag="DOCKER",
-                    params={"error": str(e)},
-                )
-            return False
-
-    # Container Command Execution Methods
-
-    async def exec_in_container(
-        self, container_id: str, command: List[str], detach: bool = False
-    ) -> Tuple[int, str, str]:
-        """Execute a command in a running container.
-
-        Args:
-            container_id: ID of the container
-            command: Command to execute as a list of strings
-            detach: Whether to run the command in detached mode
-
-        Returns:
-            Tuple of (return_code, stdout, stderr)
-        """
-        cmd = ["docker", "exec"]
-        if detach:
-            cmd.append("-d")
-        cmd.append(container_id)
-        cmd.extend(command)
-
-        try:
-            process = await asyncio.create_subprocess_exec(
-                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            stdout, stderr = await process.communicate()
-
-            return process.returncode, stdout.decode(), stderr.decode()
-        except Exception as e:
-            if self.logger:
-                self.logger.error(
-                    message="Error executing command in container: {error}",
-                    tag="DOCKER",
-                    params={"error": str(e)},
-                )
-            return -1, "", str(e)
-
-    async def start_socat_in_container(self, container_id: str) -> bool:
-        """Start socat in the container to map port 9222 to 9223.
-
-        Args:
-            container_id: ID of the container
-
-        Returns:
-            bool: True if socat started successfully, False otherwise
-        """
-        # Command to run socat as a background process
-        cmd = ["socat", "TCP-LISTEN:9223,fork", "TCP:localhost:9222"]
-
-        returncode, _, stderr = await self.exec_in_container(
-            container_id, cmd, detach=True
-        )
-
-        if returncode != 0:
-            if self.logger:
-                self.logger.error(
-                    message="Failed to start socat in container: {error}",
-                    tag="DOCKER",
-                    params={"error": stderr},
-                )
-            return False
-
-        if self.logger:
-            self.logger.debug(
-                f"Started socat in container: {container_id[:12]}", tag="DOCKER"
-            )
-
-        # Wait a moment for socat to start
-        await asyncio.sleep(1)
-        return True
-
-    async def launch_chrome_in_container(
-        self, container_id: str, browser_args: List[str]
-    ) -> bool:
-        """Launch Chrome inside the container with specified arguments.
-
-        Args:
-            container_id: ID of the container
-            browser_args: Chrome command line arguments
-
-        Returns:
-            bool: True if Chrome started successfully, False otherwise
-        """
-        # Build Chrome command
-        chrome_cmd = ["chromium"]
-        chrome_cmd.extend(browser_args)
-
-        returncode, _, stderr = await self.exec_in_container(
-            container_id, chrome_cmd, detach=True
-        )
-
-        if returncode != 0:
-            if self.logger:
-                self.logger.error(
-                    message="Failed to launch Chrome in container: {error}",
-                    tag="DOCKER",
-                    params={"error": stderr},
-                )
-            return False
-
-        if self.logger:
-            self.logger.debug(
-                f"Launched Chrome in container: {container_id[:12]}", tag="DOCKER"
-            )
-
-        return True
-
-    async def get_process_id_in_container(
-        self, container_id: str, process_name: str
-    ) -> Optional[int]:
-        """Get the process ID for a process in the container.
-
-        Args:
-            container_id: ID of the container
-            process_name: Name pattern to search for
-
-        Returns:
-            int: Process ID if found, None otherwise
-        """
-        cmd = ["pgrep", "-f", process_name]
-
-        returncode, stdout, _ = await self.exec_in_container(container_id, cmd)
-
-        if returncode == 0 and stdout.strip():
-            pid = int(stdout.strip().split("\n")[0])
-            return pid
-
-        return None
-
-    async def stop_process_in_container(self, container_id: str, pid: int) -> bool:
-        """Stop a process in the container by PID.
-
-        Args:
-            container_id: ID of the container
-            pid: Process ID to stop
-
-        Returns:
-            bool: True if process was stopped, False otherwise
-        """
-        cmd = ["kill", "-TERM", str(pid)]
-
-        returncode, _, stderr = await self.exec_in_container(container_id, cmd)
-
-        if returncode != 0:
-            if self.logger:
-                self.logger.warning(
-                    message="Failed to stop process in container: {error}",
-                    tag="DOCKER",
-                    params={"error": stderr},
-                )
-            return False
-
-        if self.logger:
-            self.logger.debug(
-                f"Stopped process {pid} in container: {container_id[:12]}", tag="DOCKER"
-            )
-
-        return True
-
-    # Network and Port Methods
-
-    async def wait_for_cdp_ready(self, host_port: int, timeout: int = 10) -> dict:
-        """Wait for the CDP endpoint to be ready.
-
-        Args:
-            host_port: Port to check for CDP endpoint
-            timeout: Maximum time to wait in seconds
-
-        Returns:
-            dict: CDP JSON config if ready, None if timeout occurred
-        """
-        import aiohttp
-
-        url = f"http://localhost:{host_port}/json/version"
-
-        for _ in range(timeout):
-            try:
-                async with aiohttp.ClientSession() as session:
-                    async with session.get(url, timeout=1) as response:
-                        if response.status == 200:
-                            if self.logger:
-                                self.logger.debug(
-                                    f"CDP endpoint ready on port {host_port}",
-                                    tag="DOCKER",
-                                )
-                            cdp_json_config = await response.json()
-                            if self.logger:
-                                self.logger.debug(
-                                    f"CDP JSON config: {cdp_json_config}", tag="DOCKER"
-                                )
-                            return cdp_json_config
-            except Exception:
-                pass
-            await asyncio.sleep(1)
-
-        if self.logger:
-            self.logger.warning(
-                f"CDP endpoint not ready on port {host_port} after {timeout}s timeout",
-                tag="DOCKER",
-            )
-        return None
-
-    def is_port_in_use(self, port: int) -> bool:
-        """Check if a port is already in use on the host.
-
-        Args:
-            port: Port number to check
-
-        Returns:
-            bool: True if port is in use, False otherwise
-        """
-        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            return s.connect_ex(("localhost", port)) == 0
-
-    def get_next_available_port(self, start_port: int = 9223) -> int:
-        """Get the next available port starting from a given port.
-
-        Args:
-            start_port: Port number to start checking from
-
-        Returns:
-            int: First available port number
-        """
-        port = start_port
-        while self.is_port_in_use(port):
-            port += 1
-        return port
-
-    # Configuration Hash Methods
-
-    def generate_config_hash(self, config_dict: Dict) -> str:
-        """Generate a hash of the configuration for container matching.
-
-        Args:
-            config_dict: Dictionary of configuration parameters
-
-        Returns:
-            str: Hash string uniquely identifying this configuration
-        """
-        # Convert to canonical JSON string and hash
-        config_json = json.dumps(config_dict, sort_keys=True)
-        return hashlib.sha256(config_json.encode()).hexdigest()

crawl4ai/browser/manager copy.py

@@ -1,177 +0,0 @@
-"""Browser manager module for Crawl4AI.
-
-This module provides a central browser management class that uses the
-strategy pattern internally while maintaining the existing API.
-It also implements a page pooling mechanism for improved performance.
-"""
-
-from typing import Optional, Tuple, List
-
-from playwright.async_api import Page, BrowserContext
-
-from ..async_logger import AsyncLogger
-from ..async_configs import BrowserConfig, CrawlerRunConfig
-
-from .strategies import (
-    BaseBrowserStrategy,
-    PlaywrightBrowserStrategy,
-    CDPBrowserStrategy,
-    BuiltinBrowserStrategy,
-    DockerBrowserStrategy
-)
-
-class BrowserManager:
-    """Main interface for browser management in Crawl4AI.
-    
-    This class maintains backward compatibility with the existing implementation
-    while using the strategy pattern internally for different browser types.
-    
-    Attributes:
-        config (BrowserConfig): Configuration object containing all browser settings
-        logger: Logger instance for recording events and errors
-        browser: The browser instance
-        default_context: The default browser context
-        managed_browser: The managed browser instance
-        playwright: The Playwright instance
-        sessions: Dictionary to store session information
-        session_ttl: Session timeout in seconds
-    """
-    
-    def __init__(self, browser_config: Optional[BrowserConfig] = None, logger: Optional[AsyncLogger] = None):
-        """Initialize the BrowserManager with a browser configuration.
-        
-        Args:
-            browser_config: Configuration object containing all browser settings
-            logger: Logger instance for recording events and errors
-        """
-        self.config = browser_config or BrowserConfig()
-        self.logger = logger
-        
-        # Create strategy based on configuration
-        self.strategy = self._create_strategy()
-        
-        # Initialize state variables for compatibility with existing code
-        self.browser = None
-        self.default_context = None
-        self.managed_browser = None
-        self.playwright = None
-        
-        # For session management (from existing implementation)
-        self.sessions = {}
-        self.session_ttl = 1800  # 30 minutes
-    
-    def _create_strategy(self) -> BaseBrowserStrategy:
-        """Create appropriate browser strategy based on configuration.
-        
-        Returns:
-            BaseBrowserStrategy: The selected browser strategy
-        """
-        if self.config.browser_mode == "builtin":
-            return BuiltinBrowserStrategy(self.config, self.logger)
-        elif self.config.browser_mode == "docker":
-            if DockerBrowserStrategy is None:
-                if self.logger:
-                    self.logger.error(
-                        "Docker browser strategy requested but not available. "
-                        "Falling back to PlaywrightBrowserStrategy.",
-                        tag="BROWSER"
-                    )
-                return PlaywrightBrowserStrategy(self.config, self.logger)
-            return DockerBrowserStrategy(self.config, self.logger)
-        elif self.config.browser_mode == "cdp" or self.config.cdp_url or self.config.use_managed_browser:
-            return CDPBrowserStrategy(self.config, self.logger)
-        else:
-            return PlaywrightBrowserStrategy(self.config, self.logger)
-    
-    async def start(self):
-        """Start the browser instance and set up the default context.
-        
-        Returns:
-            self: For method chaining
-        """
-        # Start the strategy
-        await self.strategy.start()
-        
-        # Update legacy references
-        self.browser = self.strategy.browser
-        self.default_context = self.strategy.default_context
-        
-        # Set browser process reference (for CDP strategy)
-        if hasattr(self.strategy, 'browser_process'):
-            self.managed_browser = self.strategy
-        
-        # Set Playwright reference
-        self.playwright = self.strategy.playwright
-        
-        # Sync sessions if needed
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-            self.session_ttl = self.strategy.session_ttl
-        
-        return self
-    
-    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        """Get a page for the given configuration.
-        
-        Args:
-            crawlerRunConfig: Configuration object for the crawler run
-            
-        Returns:
-            Tuple of (Page, BrowserContext)
-        """
-        # Delegate to strategy
-        page, context = await self.strategy.get_page(crawlerRunConfig)
-        
-        # Sync sessions if needed
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-        
-        return page, context
-        
-    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
-        """Get multiple pages with the same configuration.
-        
-        This method efficiently creates multiple browser pages using the same configuration,
-        which is useful for parallel crawling of multiple URLs.
-        
-        Args:
-            crawlerRunConfig: Configuration for the pages
-            count: Number of pages to create
-            
-        Returns:
-            List of (Page, Context) tuples
-        """
-        # Delegate to strategy
-        pages = await self.strategy.get_pages(crawlerRunConfig, count)
-        
-        # Sync sessions if needed
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-            
-        return pages
-    
-    # Just for legacy compatibility
-    async def kill_session(self, session_id: str):
-        """Kill a browser session and clean up resources.
-        
-        Args:
-            session_id: The session ID to kill
-        """
-        # Handle kill_session via our strategy if it supports it
-        await self.strategy.kill_session(session_id)
-
-        # sync sessions if needed
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-    
-    async def close(self):
-        """Close the browser and clean up resources."""
-        # Delegate to strategy
-        await self.strategy.close()
-        
-        # Reset legacy references
-        self.browser = None
-        self.default_context = None
-        self.managed_browser = None
-        self.playwright = None
-        self.sessions = {}

crawl4ai/browser/manager.py

@@ -1,853 +0,0 @@
-"""Browser manager module for Crawl4AI.
-
-This module provides a central browser management class that uses the
-strategy pattern internally while maintaining the existing API.
-It also implements browser pooling for improved performance.
-"""
-
-import asyncio
-import hashlib
-import json
-import math
-from enum import Enum
-from typing import Dict, List, Optional, Tuple, Any
-
-from playwright.async_api import Page, BrowserContext
-
-from ..async_logger import AsyncLogger
-from ..async_configs import BrowserConfig, CrawlerRunConfig
-
-from .strategies import (
-    BaseBrowserStrategy,
-    PlaywrightBrowserStrategy,
-    CDPBrowserStrategy,
-    BuiltinBrowserStrategy,
-    DockerBrowserStrategy
-)
-
-class UnavailableBehavior(Enum):
-    """Behavior when no browser is available."""
-    ON_DEMAND = "on_demand"  # Create new browser on demand
-    PENDING = "pending"      # Wait until a browser is available
-    EXCEPTION = "exception"  # Raise an exception
-
-
-class BrowserManager:
-    """Main interface for browser management and pooling in Crawl4AI.
-    
-    This class maintains backward compatibility with the existing implementation
-    while using the strategy pattern internally for different browser types.
-    It also implements browser pooling for improved performance.
-    
-    Attributes:
-        config (BrowserConfig): Default configuration object for browsers
-        logger (AsyncLogger): Logger instance for recording events and errors
-        browser_pool (Dict): Dictionary to store browser instances by configuration
-        browser_in_use (Dict): Dictionary to track which browsers are in use
-        request_queues (Dict): Queues for pending requests by configuration
-        unavailable_behavior (UnavailableBehavior): Behavior when no browser is available
-    """
-    
-    def __init__(
-        self, 
-        browser_config: Optional[BrowserConfig] = None, 
-        logger: Optional[AsyncLogger] = None,
-        unavailable_behavior: UnavailableBehavior = UnavailableBehavior.EXCEPTION,
-        max_browsers_per_config: int = 10,
-        max_pages_per_browser: int = 5
-        ):
-        """Initialize the BrowserManager with a browser configuration.
-        
-        Args:
-            browser_config: Configuration object containing all browser settings
-            logger: Logger instance for recording events and errors
-            unavailable_behavior: Behavior when no browser is available
-            max_browsers_per_config: Maximum number of browsers per configuration
-            max_pages_per_browser: Maximum number of pages per browser
-        """
-        self.config = browser_config or BrowserConfig()
-        self.logger = logger
-        self.unavailable_behavior = unavailable_behavior
-        self.max_browsers_per_config = max_browsers_per_config
-        self.max_pages_per_browser = max_pages_per_browser
-        
-        # Browser pool management
-        self.browser_pool = {}            # config_hash -> list of browser strategies
-        self.browser_in_use = {}          # strategy instance -> Boolean
-        self.request_queues = {}          # config_hash -> asyncio.Queue()
-        self._browser_locks = {}          # config_hash -> asyncio.Lock()
-        self._browser_pool_lock = asyncio.Lock()  # Global lock for pool modifications
-        
-        # Page pool management
-        self.page_pool = {}  # (browser_config_hash, crawler_config_hash) -> list of (page, context, strategy)
-        self._page_pool_lock = asyncio.Lock()
-            
-        self.browser_page_counts = {}  # strategy instance -> current page count
-        self._page_count_lock = asyncio.Lock()  # Lock for thread-safe access to page counts
-
-        # For session management (from existing implementation)
-        self.sessions = {}
-        self.session_ttl = 1800  # 30 minutes
-
-        # For legacy compatibility
-        self.browser = None
-        self.default_context = None
-        self.managed_browser = None
-        self.playwright = None
-        self.strategy = None
-    
-    def _create_browser_config_hash(self, browser_config: BrowserConfig) -> str:
-        """Create a hash of the browser configuration for browser pooling.
-        
-        Args:
-            browser_config: Browser configuration
-            
-        Returns:
-            str: Hash of the browser configuration
-        """
-        # Convert config to dictionary, excluding any callable objects
-        config_dict = browser_config.__dict__.copy()
-        for key in list(config_dict.keys()):
-            if callable(config_dict[key]):
-                del config_dict[key]
-        
-        # Convert to canonical JSON string
-        config_json = json.dumps(config_dict, sort_keys=True, default=str)
-        
-        # Hash the JSON
-        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
-        return config_hash
-    
-    def _create_strategy(self, browser_config: BrowserConfig) -> BaseBrowserStrategy:
-        """Create appropriate browser strategy based on configuration.
-        
-        Args:
-            browser_config: Browser configuration
-            
-        Returns:
-            BaseBrowserStrategy: The selected browser strategy
-        """
-        if browser_config.browser_mode == "builtin":
-            return BuiltinBrowserStrategy(browser_config, self.logger)
-        elif browser_config.browser_mode == "docker":
-            if DockerBrowserStrategy is None:
-                if self.logger:
-                    self.logger.error(
-                        "Docker browser strategy requested but not available. "
-                        "Falling back to PlaywrightBrowserStrategy.",
-                        tag="BROWSER"
-                    )
-                return PlaywrightBrowserStrategy(browser_config, self.logger)
-            return DockerBrowserStrategy(browser_config, self.logger)
-        elif browser_config.browser_mode == "cdp" or browser_config.cdp_url or browser_config.use_managed_browser:
-            return CDPBrowserStrategy(browser_config, self.logger)
-        else:
-            return PlaywrightBrowserStrategy(browser_config, self.logger)
-    
-    async def initialize_pool(
-        self, 
-        browser_configs: List[BrowserConfig] = None,
-        browsers_per_config: int = 1,
-        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
-    ):
-        """Initialize the browser pool with multiple browser configurations.
-        
-        Args:
-            browser_configs: List of browser configurations to initialize
-            browsers_per_config: Number of browser instances per configuration
-            page_configs: Optional list of (browser_config, crawler_run_config, count) tuples
-                for pre-warming pages
-                
-        Returns:
-            self: For method chaining
-        """
-        if not browser_configs:
-            browser_configs = [self.config]
-        
-        # Calculate how many browsers we'll need based on page_configs
-        browsers_needed = {}
-        if page_configs:
-            for browser_config, _, page_count in page_configs:
-                config_hash = self._create_browser_config_hash(browser_config)
-                # Calculate browsers based on max_pages_per_browser
-                browsers_needed_for_config = math.ceil(page_count / self.max_pages_per_browser)
-                browsers_needed[config_hash] = max(
-                    browsers_needed.get(config_hash, 0),
-                    browsers_needed_for_config
-                )
-        
-        # Adjust browsers_per_config if needed to ensure enough capacity
-        config_browsers_needed = {}
-        for browser_config in browser_configs:
-            config_hash = self._create_browser_config_hash(browser_config)
-            
-            # Estimate browsers needed based on page requirements
-            browsers_for_config = browsers_per_config
-            if config_hash in browsers_needed:
-                browsers_for_config = max(browsers_for_config, browsers_needed[config_hash])
-            
-            config_browsers_needed[config_hash] = browsers_for_config
-            
-            # Update max_browsers_per_config if needed
-            if browsers_for_config > self.max_browsers_per_config:
-                self.max_browsers_per_config = browsers_for_config
-                if self.logger:
-                    self.logger.info(
-                        f"Increased max_browsers_per_config to {browsers_for_config} to accommodate page requirements",
-                        tag="POOL"
-                    )
-        
-        # Initialize locks and queues for each config
-        async with self._browser_pool_lock:
-            for browser_config in browser_configs:
-                config_hash = self._create_browser_config_hash(browser_config)
-                
-                # Initialize lock for this config if needed
-                if config_hash not in self._browser_locks:
-                    self._browser_locks[config_hash] = asyncio.Lock()
-                
-                # Initialize queue for this config if needed
-                if config_hash not in self.request_queues:
-                    self.request_queues[config_hash] = asyncio.Queue()
-                
-                # Initialize pool for this config if needed
-                if config_hash not in self.browser_pool:
-                    self.browser_pool[config_hash] = []
-        
-        # Create browser instances for each configuration in parallel
-        browser_tasks = []
-        
-        for browser_config in browser_configs:
-            config_hash = self._create_browser_config_hash(browser_config)
-            browsers_to_create = config_browsers_needed.get(
-                config_hash, 
-                browsers_per_config
-            ) - len(self.browser_pool.get(config_hash, []))
-            
-            if browsers_to_create <= 0:
-                continue
-                
-            for _ in range(browsers_to_create):
-                # Create a task for each browser initialization
-                task = self._create_and_add_browser(browser_config, config_hash)
-                browser_tasks.append(task)
-        
-        # Wait for all browser initializations to complete
-        if browser_tasks:
-            if self.logger:
-                self.logger.info(f"Initializing {len(browser_tasks)} browsers in parallel...", tag="POOL")
-            await asyncio.gather(*browser_tasks)
-        
-        # Pre-warm pages if requested
-        if page_configs:
-            page_tasks = []
-            for browser_config, crawler_run_config, count in page_configs:
-                task = self._prewarm_pages(browser_config, crawler_run_config, count)
-                page_tasks.append(task)
-            
-            if page_tasks:
-                if self.logger:
-                    self.logger.info(f"Pre-warming pages with {len(page_tasks)} configurations...", tag="POOL")
-                await asyncio.gather(*page_tasks)
-        
-        # Update legacy references
-        if self.browser_pool and next(iter(self.browser_pool.values()), []):
-            strategy = next(iter(self.browser_pool.values()))[0]
-            self.strategy = strategy
-            self.browser = strategy.browser
-            self.default_context = strategy.default_context
-            self.playwright = strategy.playwright
-        
-        return self
-
-    async def _create_and_add_browser(self, browser_config: BrowserConfig, config_hash: str):
-        """Create and add a browser to the pool.
-        
-        Args:
-            browser_config: Browser configuration
-            config_hash: Hash of the configuration
-        """
-        try:
-            strategy = self._create_strategy(browser_config)
-            await strategy.start()
-            
-            async with self._browser_pool_lock:
-                if config_hash not in self.browser_pool:
-                    self.browser_pool[config_hash] = []
-                self.browser_pool[config_hash].append(strategy)
-                self.browser_in_use[strategy] = False
-            
-            if self.logger:
-                self.logger.debug(
-                    f"Added browser to pool: {browser_config.browser_type} "
-                    f"({browser_config.browser_mode})", 
-                    tag="POOL"
-                )
-        except Exception as e:
-            if self.logger:
-                self.logger.error(
-                    f"Failed to create browser: {str(e)}", 
-                    tag="POOL"
-                )
-            raise
-
-    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
-        """Create a signature hash from crawler configuration.
-        
-        Args:
-            crawlerRunConfig: Crawler run configuration
-            
-        Returns:
-            str: Hash of the crawler configuration
-        """
-        config_dict = crawlerRunConfig.__dict__.copy()
-        # Exclude items that do not affect page creation
-        ephemeral_keys = [
-            "session_id",
-            "js_code",
-            "scraping_strategy",
-            "extraction_strategy",
-            "chunking_strategy",
-            "cache_mode",
-            "content_filter",
-            "semaphore_count",
-            "url"
-        ]
-        for key in ephemeral_keys:
-            if key in config_dict:
-                del config_dict[key]
-                
-        # Convert to canonical JSON string
-        config_json = json.dumps(config_dict, sort_keys=True, default=str)
-
-        # Hash the JSON
-        config_hash = hashlib.sha256(config_json.encode("utf-8")).hexdigest()
-        return config_hash            
-
-    async def _prewarm_pages(
-        self, 
-        browser_config: BrowserConfig, 
-        crawler_run_config: CrawlerRunConfig, 
-        count: int
-    ):
-        """Pre-warm pages for a specific configuration.
-        
-        Args:
-            browser_config: Browser configuration
-            crawler_run_config: Crawler run configuration
-            count: Number of pages to pre-warm
-        """
-        try:
-            # Create individual page tasks and run them in parallel
-            browser_config_hash = self._create_browser_config_hash(browser_config)
-            crawler_config_hash = self._make_config_signature(crawler_run_config)            
-            async def get_single_page():
-                strategy = await self.get_available_browser(browser_config)
-                try:
-                    page, context = await strategy.get_page(crawler_run_config)
-                    # Store config hashes on the page object for later retrieval
-                    setattr(page, "_browser_config_hash", browser_config_hash)
-                    setattr(page, "_crawler_config_hash", crawler_config_hash)                    
-                    return page, context, strategy
-                except Exception as e:
-                    # Release the browser back to the pool
-                    await self.release_browser(strategy, browser_config)
-                    raise e
-                
-            # Create tasks for parallel execution
-            page_tasks = [get_single_page() for _ in range(count)]
-            
-            # Execute all page creation tasks in parallel
-            pages_contexts_strategies = await asyncio.gather(*page_tasks)
-            
-            # Add pages to the page pool
-            browser_config_hash = self._create_browser_config_hash(browser_config)
-            crawler_config_hash = self._make_config_signature(crawler_run_config)
-            pool_key = (browser_config_hash, crawler_config_hash)
-            
-            async with self._page_pool_lock:
-                if pool_key not in self.page_pool:
-                    self.page_pool[pool_key] = []
-                
-                # Add all pages to the pool
-                self.page_pool[pool_key].extend(pages_contexts_strategies)
-                
-            if self.logger:
-                self.logger.debug(
-                    f"Pre-warmed {count} pages in parallel with config {crawler_run_config}", 
-                    tag="POOL"
-                )
-        except Exception as e:
-            if self.logger:
-                self.logger.error(
-                    f"Failed to pre-warm pages: {str(e)}", 
-                    tag="POOL"
-                )
-            raise
-    
-    async def get_available_browser(
-        self, 
-        browser_config: Optional[BrowserConfig] = None
-    ) -> BaseBrowserStrategy:
-        """Get an available browser from the pool for the given configuration.
-        
-        Args:
-            browser_config: Browser configuration to match
-            
-        Returns:
-            BaseBrowserStrategy: An available browser strategy
-            
-        Raises:
-            Exception: If no browser is available and behavior is EXCEPTION
-        """
-        browser_config = browser_config or self.config
-        config_hash = self._create_browser_config_hash(browser_config)
-        
-        async with self._browser_locks.get(config_hash, asyncio.Lock()):
-            # Check if we have browsers for this config
-            if config_hash not in self.browser_pool or not self.browser_pool[config_hash]:
-                if self.unavailable_behavior == UnavailableBehavior.ON_DEMAND:
-                    # Create a new browser on demand
-                    if self.logger:
-                        self.logger.info(
-                            f"1> Creating new browser on demand for config {config_hash[:8]}",
-                            tag="POOL"
-                        )
-                    
-                    # Initialize pool for this config if needed
-                    async with self._browser_pool_lock:
-                        if config_hash not in self.browser_pool:
-                            self.browser_pool[config_hash] = []
-                        
-                        strategy = self._create_strategy(browser_config)
-                        await strategy.start()
-
-                        self.browser_pool[config_hash].append(strategy)
-                        self.browser_in_use[strategy] = False
-
-                elif self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
-                    raise Exception(f"No browsers available for configuration {config_hash[:8]}")
-            
-            # Check for an available browser with capacity in the pool
-            for strategy in self.browser_pool[config_hash]:
-                # Check if this browser has capacity for more pages
-                async with self._page_count_lock:
-                    current_pages = self.browser_page_counts.get(strategy, 0)
-
-                    if current_pages < self.max_pages_per_browser:
-                        # Increment the page count
-                        self.browser_page_counts[strategy] = current_pages + 1
-
-                        self.browser_in_use[strategy] = True
-
-                        # Get browser information for better logging
-                        browser_type = getattr(strategy.config, 'browser_type', 'unknown')
-                        browser_mode = getattr(strategy.config, 'browser_mode', 'unknown')
-                        strategy_id = id(strategy)  # Use object ID as a unique identifier
-
-                        if self.logger:
-                            self.logger.debug(
-                                f"Selected browser #{strategy_id} ({browser_type}/{browser_mode}) - " 
-                                f"pages: {current_pages+1}/{self.max_pages_per_browser}",
-                                tag="POOL"
-                            )                        
-
-                        return strategy
-            
-            # All browsers are at capacity or in use
-            if self.unavailable_behavior == UnavailableBehavior.ON_DEMAND:
-                # Check if we've reached the maximum number of browsers
-                if len(self.browser_pool[config_hash]) >= self.max_browsers_per_config:
-                    if self.logger:
-                        self.logger.warning(
-                            f"Maximum browsers reached for config {config_hash[:8]} and all at page capacity",
-                            tag="POOL"
-                        )
-                    if self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
-                        raise Exception("Maximum browsers reached and all at page capacity")
-                
-                # Create a new browser on demand
-                if self.logger:
-                    self.logger.info(
-                        f"2> Creating new browser on demand for config {config_hash[:8]}",
-                        tag="POOL"
-                    )
-                
-                strategy = self._create_strategy(browser_config)
-                await strategy.start()
-                
-                async with self._browser_pool_lock:
-                    self.browser_pool[config_hash].append(strategy)
-                    self.browser_in_use[strategy] = True
-                
-                return strategy
-            
-            # If we get here, either behavior is EXCEPTION or PENDING
-            if self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
-                raise Exception(f"All browsers in use or at page capacity for configuration {config_hash[:8]}")
-            
-            # For PENDING behavior, set up waiting mechanism
-            if config_hash not in self.request_queues:
-                self.request_queues[config_hash] = asyncio.Queue()
-            
-            # Create a future to wait on
-            future = asyncio.Future()
-            await self.request_queues[config_hash].put(future)
-            
-            if self.logger:
-                self.logger.debug(
-                    f"Waiting for available browser for config {config_hash[:8]}",
-                    tag="POOL"
-                )
-            
-            # Wait for a browser to become available
-            strategy = await future
-            return strategy
-
-    async def get_page(
-        self, 
-        crawlerRunConfig: CrawlerRunConfig,
-        browser_config: Optional[BrowserConfig] = None
-    ) -> Tuple[Page, BrowserContext, BaseBrowserStrategy]:
-        """Get a page from the browser pool."""
-        browser_config = browser_config or self.config
-        
-        # Check if we have a pre-warmed page available
-        browser_config_hash = self._create_browser_config_hash(browser_config)
-        crawler_config_hash = self._make_config_signature(crawlerRunConfig)
-        pool_key = (browser_config_hash, crawler_config_hash)
-        
-        # Try to get a page from the pool
-        async with self._page_pool_lock:
-            if pool_key in self.page_pool and self.page_pool[pool_key]:
-                # Get a page from the pool
-                page, context, strategy = self.page_pool[pool_key].pop()
-                
-                # Mark browser as in use (it already is, but ensure consistency)
-                self.browser_in_use[strategy] = True
-                
-                if self.logger:
-                    self.logger.debug(
-                        f"Using pre-warmed page for config {crawler_config_hash[:8]}", 
-                        tag="POOL"
-                    )
-                    
-                # Note: We don't increment page count since it was already counted when created
-                
-                return page, context, strategy
-        
-        # No pre-warmed page available, create a new one
-        # get_available_browser already increments the page count
-        strategy = await self.get_available_browser(browser_config)
-        
-        try:
-            # Get a page from the browser
-            page, context = await strategy.get_page(crawlerRunConfig)
-            
-            # Store config hashes on the page object for later retrieval
-            setattr(page, "_browser_config_hash", browser_config_hash)
-            setattr(page, "_crawler_config_hash", crawler_config_hash)
-            
-            return page, context, strategy
-        except Exception as e:
-            # Release the browser back to the pool and decrement the page count
-            await self.release_browser(strategy, browser_config, decrement_page_count=True)
-            raise e
-        
-    async def release_page(
-        self, 
-        page: Page, 
-        strategy: BaseBrowserStrategy, 
-        browser_config: Optional[BrowserConfig] = None,
-        keep_alive: bool = True,
-        return_to_pool: bool = True
-    ):
-        """Release a page back to the pool."""
-        browser_config = browser_config or self.config
-
-        page_url = page.url if page else None
-        
-        # If not keeping the page alive, close it and decrement count
-        if not keep_alive:
-            try:
-                await page.close()
-            except Exception as e:
-                if self.logger:
-                    self.logger.error(
-                        f"Error closing page: {str(e)}",
-                        tag="POOL"
-                    )
-            # Release the browser with page count decrement
-            await self.release_browser(strategy, browser_config, decrement_page_count=True)
-            return
-        
-        # If returning to pool
-        if return_to_pool:
-            # Get the configuration hashes from the page object
-            browser_config_hash = getattr(page, "_browser_config_hash", None)
-            crawler_config_hash = getattr(page, "_crawler_config_hash", None)
-            
-            if browser_config_hash and crawler_config_hash:
-                pool_key = (browser_config_hash, crawler_config_hash)
-                
-                async with self._page_pool_lock:
-                    if pool_key not in self.page_pool:
-                        self.page_pool[pool_key] = []
-                    
-                    # Add page back to the pool
-                    self.page_pool[pool_key].append((page, page.context, strategy))
-                    
-                    if self.logger:
-                        self.logger.debug(
-                            f"Returned page to pool for config {crawler_config_hash[:8]}, url: {page_url}", 
-                            tag="POOL"
-                        )
-                    
-                    # Note: We don't decrement the page count here since the page is still "in use" 
-                    # from the browser's perspective, just in our pool
-                    return
-            else:
-                # If we can't identify the configuration, log a warning
-                if self.logger:
-                    self.logger.warning(
-                        "Cannot return page to pool - missing configuration hashes", 
-                        tag="POOL"
-                    )
-        
-        # If we got here, we couldn't return to pool, so just release the browser
-        await self.release_browser(strategy, browser_config, decrement_page_count=True)
-    
-    async def release_browser(
-        self, 
-        strategy: BaseBrowserStrategy, 
-        browser_config: Optional[BrowserConfig] = None,
-        decrement_page_count: bool = True
-    ):
-        """Release a browser back to the pool."""
-        browser_config = browser_config or self.config
-        config_hash = self._create_browser_config_hash(browser_config)
-        
-        # Decrement page count
-        if decrement_page_count:
-            async with self._page_count_lock:
-                current_count = self.browser_page_counts.get(strategy, 1)
-                self.browser_page_counts[strategy] = max(0, current_count - 1)
-                
-                if self.logger:
-                    self.logger.debug(
-                        f"Decremented page count for browser (now: {self.browser_page_counts[strategy]})",
-                        tag="POOL"
-                    )
-        
-        # Mark as not in use
-        self.browser_in_use[strategy] = False
-        
-        # Process any waiting requests
-        if config_hash in self.request_queues and not self.request_queues[config_hash].empty():
-            future = await self.request_queues[config_hash].get()
-            if not future.done():
-                future.set_result(strategy)
-
-    async def get_pages(
-        self, 
-        crawlerRunConfig: CrawlerRunConfig, 
-        count: int = 1,
-        browser_config: Optional[BrowserConfig] = None
-    ) -> List[Tuple[Page, BrowserContext, BaseBrowserStrategy]]:
-        """Get multiple pages from the browser pool.
-        
-        Args:
-            crawlerRunConfig: Configuration for the crawler run
-            count: Number of pages to get
-            browser_config: Browser configuration to use
-            
-        Returns:
-            List of (Page, Context, Strategy) tuples
-        """
-        results = []
-        for _ in range(count):
-            try:
-                result = await self.get_page(crawlerRunConfig, browser_config)
-                results.append(result)
-            except Exception as e:
-                # Release any pages we've already gotten
-                for page, _, strategy in results:
-                    await self.release_page(page, strategy, browser_config)
-                raise e
-        
-        return results
-
-    async def get_page_pool_status(self) -> Dict[str, Any]:
-        """Get information about the page pool status.
-        
-        Returns:
-            Dict with page pool status information
-        """
-        status = {
-            "total_pooled_pages": 0,
-            "configs": {}
-        }
-        
-        async with self._page_pool_lock:
-            for (browser_hash, crawler_hash), pages in self.page_pool.items():
-                config_key = f"{browser_hash[:8]}_{crawler_hash[:8]}"
-                status["configs"][config_key] = len(pages)
-                status["total_pooled_pages"] += len(pages)
-        
-        if self.logger:
-            self.logger.debug(
-                f"Page pool status: {status['total_pooled_pages']} pages available",
-                tag="POOL"
-            )
-        
-        return status
-
-    async def get_pool_status(self) -> Dict[str, Any]:
-        """Get information about the browser pool status.
-        
-        Returns:
-            Dict with pool status information
-        """
-        status = {
-            "total_browsers": 0,
-            "browsers_in_use": 0,
-            "total_pages": 0,
-            "configs": {}
-        }
-        
-        for config_hash, strategies in self.browser_pool.items():
-            config_pages = 0
-            in_use = 0
-            
-            for strategy in strategies:
-                is_in_use = self.browser_in_use.get(strategy, False)
-                if is_in_use:
-                    in_use += 1
-                
-                # Get page count for this browser
-                try:
-                    page_count = len(await strategy.get_opened_pages())
-                    config_pages += page_count
-                except Exception as e:
-                    if self.logger:
-                        self.logger.error(f"Error getting page count: {str(e)}", tag="POOL")
-            
-            config_status = {
-                "total_browsers": len(strategies),
-                "browsers_in_use": in_use,
-                "pages_open": config_pages,
-                "waiting_requests": self.request_queues.get(config_hash, asyncio.Queue()).qsize(),
-                "max_capacity": len(strategies) * self.max_pages_per_browser,
-                "utilization_pct": round((config_pages / (len(strategies) * self.max_pages_per_browser)) * 100, 1) 
-                    if strategies else 0
-            }
-            
-            status["configs"][config_hash] = config_status
-            status["total_browsers"] += config_status["total_browsers"]
-            status["browsers_in_use"] += config_status["browsers_in_use"]
-            status["total_pages"] += config_pages
-        
-        # Add overall utilization
-        if status["total_browsers"] > 0:
-            max_capacity = status["total_browsers"] * self.max_pages_per_browser
-            status["overall_utilization_pct"] = round((status["total_pages"] / max_capacity) * 100, 1)
-        else:
-            status["overall_utilization_pct"] = 0
-        
-        return status
-
-    async def start(self):
-        """Start at least one browser instance in the pool.
-        
-        This method is kept for backward compatibility.
-        
-        Returns:
-            self: For method chaining
-        """
-        await self.initialize_pool([self.config], 1)
-        return self
-        
-    async def kill_session(self, session_id: str):
-        """Kill a browser session and clean up resources.
-        
-        Delegated to the strategy. This method is kept for backward compatibility.
-        
-        Args:
-            session_id: The session ID to kill
-        """
-        if not self.strategy:
-            return
-            
-        await self.strategy.kill_session(session_id)
-        
-        # Sync sessions
-        if hasattr(self.strategy, 'sessions'):
-            self.sessions = self.strategy.sessions
-    
-    async def close(self):
-        """Close all browsers in the pool and clean up resources."""
-        # Close all browsers in the pool
-        for strategies in self.browser_pool.values():
-            for strategy in strategies:
-                try:
-                    await strategy.close()
-                except Exception as e:
-                    if self.logger:
-                        self.logger.error(
-                            f"Error closing browser: {str(e)}",
-                            tag="POOL"
-                        )
-        
-        # Clear pool data
-        self.browser_pool = {}
-        self.browser_in_use = {}
-        
-        # Reset legacy references
-        self.browser = None
-        self.default_context = None
-        self.managed_browser = None
-        self.playwright = None
-        self.strategy = None
-        self.sessions = {}
-
-
-async def create_browser_manager(
-    browser_config: Optional[BrowserConfig] = None,
-    logger: Optional[AsyncLogger] = None,
-    unavailable_behavior: UnavailableBehavior = UnavailableBehavior.EXCEPTION,
-    max_browsers_per_config: int = 10,
-    initial_pool_size: int = 1,
-    page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
-) -> BrowserManager:
-    """Factory function to create and initialize a BrowserManager.
-    
-    Args:
-        browser_config: Configuration for the browsers
-        logger: Logger for recording events
-        unavailable_behavior: Behavior when no browser is available
-        max_browsers_per_config: Maximum browsers per configuration
-        initial_pool_size: Initial number of browsers per configuration
-        page_configs: Optional configurations for pre-warming pages
-        
-    Returns:
-        Initialized BrowserManager
-    """
-    manager = BrowserManager(
-        browser_config=browser_config,
-        logger=logger,
-        unavailable_behavior=unavailable_behavior,
-        max_browsers_per_config=max_browsers_per_config
-    )
-    
-    await manager.initialize_pool(
-        [browser_config] if browser_config else None,
-        initial_pool_size,
-        page_configs
-    )
-    
-    return manager
-
-
-
-
-

crawl4ai/browser/models.py

@@ -1,143 +0,0 @@
-"""Docker configuration module for Crawl4AI browser automation.
-
-This module provides configuration classes for Docker-based browser automation,
-allowing flexible configuration of Docker containers for browsing.
-"""
-
-from typing import Dict, List, Optional
-
-
-class DockerConfig:
-    """Configuration for Docker-based browser automation.
-    
-    This class contains Docker-specific settings to avoid cluttering BrowserConfig.
-    
-    Attributes:
-        mode (str): Docker operation mode - "connect" or "launch".
-            - "connect": Uses a container with Chrome already running
-            - "launch": Dynamically configures and starts Chrome in container
-        image (str): Docker image to use. If None, defaults from DockerUtils are used.
-        registry_file (str): Path to container registry file for persistence.
-        persistent (bool): Keep container running after browser closes.
-        remove_on_exit (bool): Remove container on exit when not persistent.
-        network (str): Docker network to use.
-        volumes (List[str]): Volume mappings (e.g., ["host_path:container_path"]).
-        env_vars (Dict[str, str]): Environment variables to set in container.
-        extra_args (List[str]): Additional docker run arguments.
-        host_port (int): Host port to map to container's 9223 port.
-        user_data_dir (str): Path to user data directory on host.
-        container_user_data_dir (str): Path to user data directory in container.
-    """
-    
-    def __init__(
-        self,
-        mode: str = "connect",                     # "connect" or "launch" 
-        image: Optional[str] = None,               # Docker image to use
-        registry_file: Optional[str] = None,       # Path to registry file
-        persistent: bool = False,                  # Keep container running after browser closes
-        remove_on_exit: bool = True,               # Remove container on exit when not persistent
-        network: Optional[str] = None,             # Docker network to use
-        volumes: List[str] = None,                 # Volume mappings
-        cpu_limit: float = 1.0,                    # CPU limit for the container
-        memory_limit: str = "1.5g",                # Memory limit for the container
-        env_vars: Dict[str, str] = None,           # Environment variables
-        host_port: Optional[int] = None,           # Host port to map to container's 9223
-        user_data_dir: Optional[str] = None,       # Path to user data directory on host
-        container_user_data_dir: str = "/data",    # Path to user data directory in container
-        extra_args: List[str] = None,              # Additional docker run arguments
-    ):
-        """Initialize Docker configuration.
-        
-        Args:
-            mode: Docker operation mode ("connect" or "launch")
-            image: Docker image to use
-            registry_file: Path to container registry file
-            persistent: Whether to keep container running after browser closes
-            remove_on_exit: Whether to remove container on exit when not persistent
-            network: Docker network to use
-            volumes: Volume mappings as list of strings
-            cpu_limit: CPU limit for the container
-            memory_limit: Memory limit for the container
-            env_vars: Environment variables as dictionary
-            extra_args: Additional docker run arguments
-            host_port: Host port to map to container's 9223
-            user_data_dir: Path to user data directory on host
-            container_user_data_dir: Path to user data directory in container
-        """
-        self.mode = mode
-        self.image = image  # If None, defaults will be used from DockerUtils
-        self.registry_file = registry_file
-        self.persistent = persistent
-        self.remove_on_exit = remove_on_exit
-        self.network = network
-        self.volumes = volumes or []
-        self.cpu_limit = cpu_limit
-        self.memory_limit = memory_limit
-        self.env_vars = env_vars or {}
-        self.extra_args = extra_args or []
-        self.host_port = host_port
-        self.user_data_dir = user_data_dir
-        self.container_user_data_dir = container_user_data_dir
-    
-    def to_dict(self) -> Dict:
-        """Convert this configuration to a dictionary.
-        
-        Returns:
-            Dictionary representation of this configuration
-        """
-        return {
-            "mode": self.mode,
-            "image": self.image,
-            "registry_file": self.registry_file,
-            "persistent": self.persistent,
-            "remove_on_exit": self.remove_on_exit,
-            "network": self.network,
-            "volumes": self.volumes,
-            "cpu_limit": self.cpu_limit,
-            "memory_limit": self.memory_limit,
-            "env_vars": self.env_vars,
-            "extra_args": self.extra_args,
-            "host_port": self.host_port,
-            "user_data_dir": self.user_data_dir,
-            "container_user_data_dir": self.container_user_data_dir
-        }
-        
-    @staticmethod
-    def from_kwargs(kwargs: Dict) -> "DockerConfig":
-        """Create a DockerConfig from a dictionary of keyword arguments.
-        
-        Args:
-            kwargs: Dictionary of configuration options
-            
-        Returns:
-            New DockerConfig instance
-        """
-        return DockerConfig(
-            mode=kwargs.get("mode", "connect"),
-            image=kwargs.get("image"),
-            registry_file=kwargs.get("registry_file"),
-            persistent=kwargs.get("persistent", False),
-            remove_on_exit=kwargs.get("remove_on_exit", True),
-            network=kwargs.get("network"),
-            volumes=kwargs.get("volumes"),
-            cpu_limit=kwargs.get("cpu_limit", 1.0),
-            memory_limit=kwargs.get("memory_limit", "1.5g"),
-            env_vars=kwargs.get("env_vars"),
-            extra_args=kwargs.get("extra_args"),
-            host_port=kwargs.get("host_port"),
-            user_data_dir=kwargs.get("user_data_dir"),
-            container_user_data_dir=kwargs.get("container_user_data_dir", "/data")
-        )
-        
-    def clone(self, **kwargs) -> "DockerConfig":
-        """Create a copy of this configuration with updated values.
-        
-        Args:
-            **kwargs: Key-value pairs of configuration options to update
-            
-        Returns:
-            DockerConfig: A new instance with the specified updates
-        """
-        config_dict = self.to_dict()
-        config_dict.update(kwargs)
-        return DockerConfig.from_kwargs(config_dict)

crawl4ai/browser/profiles.py

@@ -1,457 +0,0 @@
-"""Browser profile management module for Crawl4AI.
-
-This module provides functionality for creating and managing browser profiles
-that can be used for authenticated browsing.
-"""
-
-import os
-import asyncio
-import signal
-import sys
-import datetime
-import uuid
-import shutil
-from typing import List, Dict, Optional, Any
-from colorama import Fore, Style, init
-
-from ..async_configs import BrowserConfig
-from ..async_logger import AsyncLogger, AsyncLoggerBase
-from ..utils import get_home_folder
-
-class BrowserProfileManager:
-    """Manages browser profiles for Crawl4AI.
-    
-    This class provides functionality to create and manage browser profiles
-    that can be used for authenticated browsing with Crawl4AI.
-    
-    Profiles are stored by default in ~/.crawl4ai/profiles/
-    """
-    
-    def __init__(self, logger: Optional[AsyncLoggerBase] = None):
-        """Initialize the BrowserProfileManager.
-        
-        Args:
-            logger: Logger for outputting messages. If None, a default AsyncLogger is created.
-        """
-        # Initialize colorama for colorful terminal output
-        init()
-        
-        # Create a logger if not provided
-        if logger is None:
-            self.logger = AsyncLogger(verbose=True)
-        elif not isinstance(logger, AsyncLoggerBase):
-            self.logger = AsyncLogger(verbose=True)
-        else:
-            self.logger = logger
-            
-        # Ensure profiles directory exists
-        self.profiles_dir = os.path.join(get_home_folder(), "profiles")
-        os.makedirs(self.profiles_dir, exist_ok=True)
-    
-    async def create_profile(self, 
-                          profile_name: Optional[str] = None, 
-                          browser_config: Optional[BrowserConfig] = None) -> Optional[str]:
-        """Create a browser profile interactively.
-        
-        Args:
-            profile_name: Name for the profile. If None, a name is generated.
-            browser_config: Configuration for the browser. If None, a default configuration is used.
-                
-        Returns:
-            Path to the created profile directory, or None if creation failed
-        """
-        # Create default browser config if none provided
-        if browser_config is None:
-            browser_config = BrowserConfig(
-                browser_type="chromium",
-                headless=False,  # Must be visible for user interaction
-                verbose=True
-            )
-        else:
-            # Ensure headless is False for user interaction
-            browser_config.headless = False
-            
-        # Generate profile name if not provided
-        if not profile_name:
-            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-            profile_name = f"profile_{timestamp}_{uuid.uuid4().hex[:6]}"
-            
-        # Sanitize profile name (replace spaces and special chars)
-        profile_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in profile_name)
-        
-        # Set user data directory
-        profile_path = os.path.join(self.profiles_dir, profile_name)
-        os.makedirs(profile_path, exist_ok=True)
-        
-        # Print instructions for the user with colorama formatting
-        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
-        self.logger.info(f"\n{border}", tag="PROFILE")
-        self.logger.info(f"Creating browser profile: {Fore.GREEN}{profile_name}{Style.RESET_ALL}", tag="PROFILE")
-        self.logger.info(f"Profile directory: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
-        
-        self.logger.info("\nInstructions:", tag="PROFILE")
-        self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE")
-        self.logger.info(f"2. {Fore.CYAN}Log in to websites{Style.RESET_ALL}, configure settings, etc. as needed.", tag="PROFILE")
-        self.logger.info(f"3. When you're done, {Fore.YELLOW}press 'q' in this terminal{Style.RESET_ALL} to close the browser.", tag="PROFILE")
-        self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE")
-        self.logger.info(f"{border}\n", tag="PROFILE")
-        
-        # Import the necessary classes with local imports to avoid circular references
-        from .strategies import CDPBrowserStrategy
-        
-        # Set browser config to use the profile path
-        browser_config.user_data_dir = profile_path
-        
-        # Create a CDP browser strategy for the profile creation
-        browser_strategy = CDPBrowserStrategy(browser_config, self.logger)
-        
-        # Set up signal handlers to ensure cleanup on interrupt
-        original_sigint = signal.getsignal(signal.SIGINT)
-        original_sigterm = signal.getsignal(signal.SIGTERM)
-        
-        # Define cleanup handler for signals
-        async def cleanup_handler(sig, frame):
-            self.logger.warning("\nCleaning up browser process...", tag="PROFILE")
-            await browser_strategy.close()
-            # Restore original signal handlers
-            signal.signal(signal.SIGINT, original_sigint)
-            signal.signal(signal.SIGTERM, original_sigterm)
-            if sig == signal.SIGINT:
-                self.logger.error("Profile creation interrupted. Profile may be incomplete.", tag="PROFILE")
-                sys.exit(1)
-                
-        # Set signal handlers
-        def sigint_handler(sig, frame):
-            asyncio.create_task(cleanup_handler(sig, frame))
-        
-        signal.signal(signal.SIGINT, sigint_handler)
-        signal.signal(signal.SIGTERM, sigint_handler)
-        
-        # Event to signal when user is done with the browser
-        user_done_event = asyncio.Event()
-        
-        # Run keyboard input loop in a separate task
-        async def listen_for_quit_command():
-            import termios
-            import tty
-            import select
-            
-            # First output the prompt
-            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' when you've finished using the browser...{Style.RESET_ALL}", tag="PROFILE")
-            
-            # Save original terminal settings
-            fd = sys.stdin.fileno()
-            old_settings = termios.tcgetattr(fd)
-            
-            try:
-                # Switch to non-canonical mode (no line buffering)
-                tty.setcbreak(fd)
-                
-                while True:
-                    # Check if input is available (non-blocking)
-                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
-                    if readable:
-                        key = sys.stdin.read(1)
-                        if key.lower() == 'q':
-                            self.logger.info(f"{Fore.GREEN}Closing browser and saving profile...{Style.RESET_ALL}", tag="PROFILE")
-                            user_done_event.set()
-                            return
-                    
-                    # Check if the browser process has already exited
-                    if browser_strategy.browser_process and browser_strategy.browser_process.poll() is not None:
-                        self.logger.info("Browser already closed. Ending input listener.", tag="PROFILE")
-                        user_done_event.set()
-                        return
-                        
-                    await asyncio.sleep(0.1)
-            
-            finally:
-                # Restore terminal settings 
-                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
-        
-        try:
-            # Start the browser
-            await browser_strategy.start()
-            
-            # Check if browser started successfully
-            if not browser_strategy.browser_process:
-                self.logger.error("Failed to start browser process.", tag="PROFILE")
-                return None
-            
-            self.logger.info(f"Browser launched. {Fore.CYAN}Waiting for you to finish...{Style.RESET_ALL}", tag="PROFILE") 
-            
-            # Start listening for keyboard input
-            listener_task = asyncio.create_task(listen_for_quit_command())
-            
-            # Wait for either the user to press 'q' or for the browser process to exit naturally
-            while not user_done_event.is_set() and browser_strategy.browser_process.poll() is None:
-                await asyncio.sleep(0.5)
-            
-            # Cancel the listener task if it's still running
-            if not listener_task.done():
-                listener_task.cancel()
-                try:
-                    await listener_task
-                except asyncio.CancelledError:
-                    pass
-            
-            # If the browser is still running and the user pressed 'q', terminate it
-            if browser_strategy.browser_process.poll() is None and user_done_event.is_set():
-                self.logger.info("Terminating browser process...", tag="PROFILE")
-                await browser_strategy.close()
-            
-            self.logger.success(f"Browser closed. Profile saved at: {Fore.GREEN}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
-                
-        except Exception as e:
-            self.logger.error(f"Error creating profile: {str(e)}", tag="PROFILE")
-            await browser_strategy.close()
-            return None
-        finally:
-            # Restore original signal handlers
-            signal.signal(signal.SIGINT, original_sigint)
-            signal.signal(signal.SIGTERM, original_sigterm)
-            
-            # Make sure browser is fully cleaned up
-            await browser_strategy.close()
-        
-        # Return the profile path
-        return profile_path
-    
-    def list_profiles(self) -> List[Dict[str, Any]]:
-        """List all available browser profiles.
-        
-        Returns:
-            List of dictionaries containing profile information
-        """
-        if not os.path.exists(self.profiles_dir):
-            return []
-            
-        profiles = []
-        
-        for name in os.listdir(self.profiles_dir):
-            profile_path = os.path.join(self.profiles_dir, name)
-            
-            # Skip if not a directory
-            if not os.path.isdir(profile_path):
-                continue
-                
-            # Check if this looks like a valid browser profile
-            # For Chromium: Look for Preferences file
-            # For Firefox: Look for prefs.js file
-            is_valid = False
-            
-            if os.path.exists(os.path.join(profile_path, "Preferences")) or \
-               os.path.exists(os.path.join(profile_path, "Default", "Preferences")):
-                is_valid = "chromium"
-            elif os.path.exists(os.path.join(profile_path, "prefs.js")):
-                is_valid = "firefox"
-                
-            if is_valid:
-                # Get creation time
-                created = datetime.datetime.fromtimestamp(
-                    os.path.getctime(profile_path)
-                )
-                
-                profiles.append({
-                    "name": name,
-                    "path": profile_path,
-                    "created": created,
-                    "type": is_valid
-                })
-                
-        # Sort by creation time, newest first
-        profiles.sort(key=lambda x: x["created"], reverse=True)
-        
-        return profiles
-    
-    def get_profile_path(self, profile_name: str) -> Optional[str]:
-        """Get the full path to a profile by name.
-        
-        Args:
-            profile_name: Name of the profile (not the full path)
-            
-        Returns:
-            Full path to the profile directory, or None if not found
-        """
-        profile_path = os.path.join(self.profiles_dir, profile_name)
-        
-        # Check if path exists and is a valid profile
-        if not os.path.isdir(profile_path):
-            # Check if profile_name itself is full path
-            if os.path.isabs(profile_name):
-                profile_path = profile_name
-            else:
-                return None
-        
-        # Look for profile indicators
-        is_profile = (
-            os.path.exists(os.path.join(profile_path, "Preferences")) or
-            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
-            os.path.exists(os.path.join(profile_path, "prefs.js"))
-        )
-        
-        if not is_profile:
-            return None  # Not a valid browser profile
-            
-        return profile_path
-    
-    def delete_profile(self, profile_name_or_path: str) -> bool:
-        """Delete a browser profile by name or path.
-        
-        Args:
-            profile_name_or_path: Name of the profile or full path to profile directory
-            
-        Returns:
-            True if the profile was deleted successfully, False otherwise
-        """
-        # Determine if input is a name or a path
-        if os.path.isabs(profile_name_or_path):
-            # Full path provided
-            profile_path = profile_name_or_path
-        else:
-            # Just a name provided, construct path
-            profile_path = os.path.join(self.profiles_dir, profile_name_or_path)
-        
-        # Check if path exists and is a valid profile
-        if not os.path.isdir(profile_path):
-            return False
-            
-        # Look for profile indicators
-        is_profile = (
-            os.path.exists(os.path.join(profile_path, "Preferences")) or
-            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
-            os.path.exists(os.path.join(profile_path, "prefs.js"))
-        )
-        
-        if not is_profile:
-            return False  # Not a valid browser profile
-            
-        # Delete the profile directory
-        try:
-            shutil.rmtree(profile_path)
-            return True
-        except Exception:
-            return False
-            
-    async def interactive_manager(self, crawl_callback=None):
-        """Launch an interactive profile management console.
-        
-        Args:
-            crawl_callback: Function to call when selecting option to use 
-                a profile for crawling. It will be called with (profile_path, url).
-        """
-        while True:
-            self.logger.info(f"\n{Fore.CYAN}Profile Management Options:{Style.RESET_ALL}", tag="MENU")
-            self.logger.info(f"1. {Fore.GREEN}Create a new profile{Style.RESET_ALL}", tag="MENU")
-            self.logger.info(f"2. {Fore.YELLOW}List available profiles{Style.RESET_ALL}", tag="MENU")
-            self.logger.info(f"3. {Fore.RED}Delete a profile{Style.RESET_ALL}", tag="MENU")
-            
-            # Only show crawl option if callback provided
-            if crawl_callback:
-                self.logger.info(f"4. {Fore.CYAN}Use a profile to crawl a website{Style.RESET_ALL}", tag="MENU")
-                self.logger.info(f"5. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
-                exit_option = "5"
-            else:
-                self.logger.info(f"4. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
-                exit_option = "4"
-            
-            choice = input(f"\n{Fore.CYAN}Enter your choice (1-{exit_option}): {Style.RESET_ALL}")
-            
-            if choice == "1":
-                # Create new profile
-                name = input(f"{Fore.GREEN}Enter a name for the new profile (or press Enter for auto-generated name): {Style.RESET_ALL}")
-                await self.create_profile(name or None)
-                
-            elif choice == "2":
-                # List profiles
-                profiles = self.list_profiles()
-                
-                if not profiles:
-                    self.logger.warning("  No profiles found. Create one first with option 1.", tag="PROFILES")
-                    continue
-                
-                # Print profile information with colorama formatting
-                self.logger.info("\nAvailable profiles:", tag="PROFILES")
-                for i, profile in enumerate(profiles):
-                    self.logger.info(f"[{i+1}] {Fore.CYAN}{profile['name']}{Style.RESET_ALL}", tag="PROFILES")
-                    self.logger.info(f"    Path: {Fore.YELLOW}{profile['path']}{Style.RESET_ALL}", tag="PROFILES")
-                    self.logger.info(f"    Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES")
-                    self.logger.info(f"    Browser type: {profile['type']}", tag="PROFILES")
-                    self.logger.info("", tag="PROFILES")  # Empty line for spacing
-                
-            elif choice == "3":
-                # Delete profile
-                profiles = self.list_profiles()
-                if not profiles:
-                    self.logger.warning("No profiles found to delete", tag="PROFILES")
-                    continue
-                    
-                # Display numbered list
-                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
-                for i, profile in enumerate(profiles):
-                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
-                    
-                # Get profile to delete
-                profile_idx = input(f"{Fore.RED}Enter the number of the profile to delete (or 'c' to cancel): {Style.RESET_ALL}")
-                if profile_idx.lower() == 'c':
-                    continue
-                    
-                try:
-                    idx = int(profile_idx) - 1
-                    if 0 <= idx < len(profiles):
-                        profile_name = profiles[idx]["name"]
-                        self.logger.info(f"Deleting profile: {Fore.YELLOW}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
-                        
-                        # Confirm deletion
-                        confirm = input(f"{Fore.RED}Are you sure you want to delete this profile? (y/n): {Style.RESET_ALL}")
-                        if confirm.lower() == 'y':
-                            success = self.delete_profile(profiles[idx]["path"])
-                            
-                            if success:
-                                self.logger.success(f"Profile {Fore.GREEN}{profile_name}{Style.RESET_ALL} deleted successfully", tag="PROFILES")
-                            else:
-                                self.logger.error(f"Failed to delete profile {Fore.RED}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
-                    else:
-                        self.logger.error("Invalid profile number", tag="PROFILES")
-                except ValueError:
-                    self.logger.error("Please enter a valid number", tag="PROFILES")
-                    
-            elif choice == "4" and crawl_callback:
-                # Use profile to crawl a site
-                profiles = self.list_profiles()
-                if not profiles:
-                    self.logger.warning("No profiles found. Create one first.", tag="PROFILES")
-                    continue
-                    
-                # Display numbered list
-                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
-                for i, profile in enumerate(profiles):
-                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
-                    
-                # Get profile to use
-                profile_idx = input(f"{Fore.CYAN}Enter the number of the profile to use (or 'c' to cancel): {Style.RESET_ALL}")
-                if profile_idx.lower() == 'c':
-                    continue
-                    
-                try:
-                    idx = int(profile_idx) - 1
-                    if 0 <= idx < len(profiles):
-                        profile_path = profiles[idx]["path"]
-                        url = input(f"{Fore.CYAN}Enter the URL to crawl: {Style.RESET_ALL}")
-                        if url:
-                            # Call the provided crawl callback
-                            await crawl_callback(profile_path, url)
-                        else:
-                            self.logger.error("No URL provided", tag="CRAWL")
-                    else:
-                        self.logger.error("Invalid profile number", tag="PROFILES")
-                except ValueError:
-                    self.logger.error("Please enter a valid number", tag="PROFILES")
-                    
-            elif (choice == "4" and not crawl_callback) or (choice == "5" and crawl_callback):
-                # Exit
-                self.logger.info("Exiting profile management", tag="MENU")
-                break
-                
-            else:
-                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")

crawl4ai/browser/strategies/__init__.py

@@ -1,13 +0,0 @@
-from .base import BaseBrowserStrategy
-from .cdp import CDPBrowserStrategy
-from .docker_strategy import DockerBrowserStrategy
-from .playwright import PlaywrightBrowserStrategy
-from .builtin import BuiltinBrowserStrategy
-
-__all__ = [
-    "BrowserStrategy",
-    "CDPBrowserStrategy",
-    "DockerBrowserStrategy",
-    "PlaywrightBrowserStrategy",
-    "BuiltinBrowserStrategy",
-]

crawl4ai/browser/strategies/base.py

@@ -1,601 +0,0 @@
-"""Browser strategies module for Crawl4AI.
-
-This module implements the browser strategy pattern for different
-browser implementations, including Playwright, CDP, and builtin browsers.
-"""
-
-from abc import ABC, abstractmethod
-import asyncio
-import json
-import hashlib
-import os
-import time
-from typing import Optional, Tuple, List
-
-from playwright.async_api import BrowserContext, Page
-
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig, CrawlerRunConfig
-from ...config import DOWNLOAD_PAGE_TIMEOUT
-from ...js_snippet import load_js_script
-from ..utils import get_playwright
-
-
-class BaseBrowserStrategy(ABC):
-    """Base class for all browser strategies.
-    
-    This abstract class defines the interface that all browser strategies
-    must implement. It handles common functionality like context caching,
-    browser configuration, and session management.
-    """
-    
-    _playwright_instance = None
-    
-    @classmethod
-    async def get_playwright(cls):
-        """Get or create a shared Playwright instance.
-        
-        Returns:
-            Playwright: The shared Playwright instance
-        """
-        # For now I dont want Singleton pattern for Playwright
-        if cls._playwright_instance is None or True:
-            cls._playwright_instance = await get_playwright()
-        return cls._playwright_instance
-        
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the strategy with configuration and logger.
-        
-        Args:
-            config: Browser configuration
-            logger: Logger for recording events and errors
-        """
-        self.config = config
-        self.logger = logger
-        self.browser = None
-        self.default_context = None
-        
-        # Context management
-        self.contexts_by_config = {}  # config_signature -> context
-
-        self._contexts_lock = asyncio.Lock()
-        
-        # Session management
-        self.sessions = {}
-        self.session_ttl = 1800  # 30 minutes default
-        
-        # Playwright instance
-        self.playwright = None
-    
-    @abstractmethod
-    async def start(self):
-        """Start the browser.
-        
-        This method should be implemented by concrete strategies to initialize 
-        the browser in the appropriate way (direct launch, CDP connection, etc.)
-        
-        Returns:
-            self: For method chaining
-        """
-        # Base implementation gets the playwright instance
-        self.playwright = await self.get_playwright()
-        return self
-    
-    @abstractmethod
-    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        pass
-
-    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        """Get a page with specified configuration.
-        
-        This method should be implemented by concrete strategies to create
-        or retrieve a page according to their browser management approach.
-        
-        Args:
-            crawlerRunConfig: Crawler run configuration
-            
-        Returns:
-            Tuple of (Page, BrowserContext)
-        """
-        # Clean up expired sessions first
-        self._cleanup_expired_sessions()
-
-        # If a session_id is provided and we already have it, reuse that page + context
-        if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions:
-            context, page, _ = self.sessions[crawlerRunConfig.session_id]
-            # Update last-used timestamp
-            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
-            return page, context
-        
-        page, context = await self._generate_page(crawlerRunConfig)
-
-        import uuid
-        setattr(page, "guid", uuid.uuid4())
-
-        # If a session_id is specified, store this session so we can reuse later
-        if crawlerRunConfig.session_id:
-            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
-            
-        return page, context        
-        pass
-    
-    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
-        """Get multiple pages with the same configuration.
-        
-        Args:
-            crawlerRunConfig: Configuration for the pages
-            count: Number of pages to create
-            
-        Returns:
-            List of (Page, Context) tuples
-        """
-        pages = []
-        for _ in range(count):
-            page, context = await self.get_page(crawlerRunConfig)
-            pages.append((page, context))
-        return pages
-       
-    async def get_opened_pages(self) -> List[Page]:
-        """Get all opened pages in the
-        browser.
-        """
-        return [page for context in self.contexts_by_config.values() for page in context.pages]
-
-    def _build_browser_args(self) -> dict:
-        """Build browser launch arguments from config.
-        
-        Returns:
-            dict: Browser launch arguments for Playwright
-        """
-        # Define common browser arguments that improve performance and stability
-        args = [
-            "--no-sandbox",
-            "--no-first-run",
-            "--no-default-browser-check",
-            "--window-position=0,0",
-            "--ignore-certificate-errors",
-            "--ignore-certificate-errors-spki-list",
-            "--window-position=400,0",
-            "--force-color-profile=srgb",
-            "--mute-audio",
-            "--disable-gpu",
-            "--disable-gpu-compositing",
-            "--disable-software-rasterizer",
-            "--disable-dev-shm-usage",
-            "--disable-infobars",
-            "--disable-blink-features=AutomationControlled",
-            "--disable-renderer-backgrounding",
-            "--disable-ipc-flooding-protection",
-            "--disable-background-timer-throttling",
-            f"--window-size={self.config.viewport_width},{self.config.viewport_height}",
-        ]
-
-        # Define browser disable options for light mode
-        browser_disable_options = [
-            "--disable-backgrounding-occluded-windows",
-            "--disable-breakpad",
-            "--disable-client-side-phishing-detection",
-            "--disable-component-extensions-with-background-pages",
-            "--disable-default-apps",
-            "--disable-extensions",
-            "--disable-features=TranslateUI",
-            "--disable-hang-monitor",
-            "--disable-popup-blocking",
-            "--disable-prompt-on-repost",
-            "--disable-sync",
-            "--metrics-recording-only",
-            "--password-store=basic",
-            "--use-mock-keychain",
-        ]
-
-        # Apply light mode settings if enabled
-        if self.config.light_mode:
-            args.extend(browser_disable_options)
-
-        # Apply text mode settings if enabled (disables images, JS, etc)
-        if self.config.text_mode:
-            args.extend([
-                "--blink-settings=imagesEnabled=false",
-                "--disable-remote-fonts",
-                "--disable-images",
-                "--disable-javascript",
-                "--disable-software-rasterizer",
-                "--disable-dev-shm-usage",
-            ])
-
-        # Add any extra arguments from the config
-        if self.config.extra_args:
-            args.extend(self.config.extra_args)
-
-        # Build the core browser args dictionary
-        browser_args = {"headless": self.config.headless, "args": args}
-
-        # Add chrome channel if specified
-        if self.config.chrome_channel:
-            browser_args["channel"] = self.config.chrome_channel
-
-        # Configure downloads
-        if self.config.accept_downloads:
-            browser_args["downloads_path"] = self.config.downloads_path or os.path.join(
-                os.getcwd(), "downloads"
-            )
-            os.makedirs(browser_args["downloads_path"], exist_ok=True)
-
-        # Check for user data directory
-        if self.config.user_data_dir:
-            # Ensure the directory exists
-            os.makedirs(self.config.user_data_dir, exist_ok=True)
-            browser_args["user_data_dir"] = self.config.user_data_dir
-        
-        # Configure proxy settings
-        if self.config.proxy or self.config.proxy_config:
-            from playwright.async_api import ProxySettings
-
-            proxy_settings = (
-                ProxySettings(server=self.config.proxy)
-                if self.config.proxy
-                else ProxySettings(
-                    server=self.config.proxy_config.server,
-                    username=self.config.proxy_config.username,
-                    password=self.config.proxy_config.password,
-                )
-            )
-            browser_args["proxy"] = proxy_settings
-
-        return browser_args
-    
-    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
-        """Create a signature hash from configuration for context caching.
-        
-        Converts the crawlerRunConfig into a dict, excludes ephemeral fields,
-        then returns a hash of the sorted JSON. This yields a stable signature
-        that identifies configurations requiring a unique browser context.
-        
-        Args:
-            crawlerRunConfig: Crawler run configuration
-            
-        Returns:
-            str: Unique hash for this configuration
-        """
-        config_dict = crawlerRunConfig.__dict__.copy()
-        # Exclude items that do not affect browser-level setup
-        ephemeral_keys = [
-            "session_id",
-            "js_code",
-            "scraping_strategy",
-            "extraction_strategy",
-            "chunking_strategy",
-            "cache_mode",
-            "content_filter",
-            "semaphore_count",
-            "url"
-        ]
-        for key in ephemeral_keys:
-            if key in config_dict:
-                del config_dict[key]
-                
-        # Convert to canonical JSON string
-        signature_json = json.dumps(config_dict, sort_keys=True, default=str)
-
-        # Hash the JSON so we get a compact, unique string
-        signature_hash = hashlib.sha256(signature_json.encode("utf-8")).hexdigest()
-        return signature_hash
-        
-    async def create_browser_context(self, crawlerRunConfig: Optional[CrawlerRunConfig] = None) -> BrowserContext:
-        """Creates and returns a new browser context with configured settings.
-        
-        Args:
-            crawlerRunConfig: Configuration object for the crawler run
-            
-        Returns:
-            BrowserContext: Browser context object with the specified configurations
-        """
-        if not self.browser:
-            raise ValueError("Browser must be initialized before creating context")
-            
-        # Base settings
-        user_agent = self.config.headers.get("User-Agent", self.config.user_agent) 
-        viewport_settings = {
-            "width": self.config.viewport_width,
-            "height": self.config.viewport_height,
-        }
-        proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
-        
-        # Define blocked extensions for resource optimization
-        blocked_extensions = [
-            # Images
-            "jpg", "jpeg", "png", "gif", "webp", "svg", "ico", "bmp", "tiff", "psd",
-            # Fonts
-            "woff", "woff2", "ttf", "otf", "eot",
-            # Media
-            "mp4", "webm", "ogg", "avi", "mov", "wmv", "flv", "m4v", "mp3", "wav", "aac",
-            "m4a", "opus", "flac",
-            # Documents
-            "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx",
-            # Archives
-            "zip", "rar", "7z", "tar", "gz",
-            # Scripts and data
-            "xml", "swf", "wasm",
-        ]
-
-        # Common context settings
-        context_settings = {
-            "user_agent": user_agent,
-            "viewport": viewport_settings,
-            "proxy": proxy_settings,
-            "accept_downloads": self.config.accept_downloads,
-            "storage_state": self.config.storage_state,
-            "ignore_https_errors": self.config.ignore_https_errors,
-            "device_scale_factor": 1.0,
-            "java_script_enabled": self.config.java_script_enabled,
-        }
-        
-        # Apply text mode settings if enabled
-        if self.config.text_mode:
-            text_mode_settings = {
-                "has_touch": False,
-                "is_mobile": False,
-                "java_script_enabled": False,  # Disable javascript in text mode
-            }
-            # Update context settings with text mode settings
-            context_settings.update(text_mode_settings)
-            if self.logger:
-                self.logger.debug("Text mode enabled for browser context", tag="BROWSER")
-        
-        # Handle storage state properly - this is key for persistence
-        if self.config.storage_state:
-            if self.logger:
-                if isinstance(self.config.storage_state, str):
-                    self.logger.debug(f"Using storage state from file: {self.config.storage_state}", tag="BROWSER")
-                else:
-                    self.logger.debug("Using storage state from config object", tag="BROWSER")
-
-        if self.config.user_data_dir:
-            # For CDP-based browsers, storage persistence is typically handled by the user_data_dir
-            # at the browser level, but we'll create a storage_state location for Playwright as well
-            storage_path = os.path.join(self.config.user_data_dir, "storage_state.json")
-            if not os.path.exists(storage_path):
-                # Create parent directory if it doesn't exist
-                os.makedirs(os.path.dirname(storage_path), exist_ok=True)
-                with open(storage_path, "w") as f:
-                    json.dump({}, f)
-            self.config.storage_state = storage_path
-
-            if self.logger:
-                self.logger.debug(f"Using user data directory: {self.config.user_data_dir}", tag="BROWSER")
-        
-        # Apply crawler-specific configurations if provided
-        if crawlerRunConfig:
-            # Check if there is value for crawlerRunConfig.proxy_config set add that to context
-            if crawlerRunConfig.proxy_config:
-                proxy_settings = {
-                    "server": crawlerRunConfig.proxy_config.server,
-                }
-                if crawlerRunConfig.proxy_config.username:
-                    proxy_settings.update({
-                        "username": crawlerRunConfig.proxy_config.username,
-                        "password": crawlerRunConfig.proxy_config.password,
-                    })
-                context_settings["proxy"] = proxy_settings
-                
-        # Create and return the context
-        try:
-            # Create the context with appropriate settings
-            context = await self.browser.new_context(**context_settings)
-            
-            # Apply text mode resource blocking if enabled
-            if self.config.text_mode:
-                # Create and apply route patterns for each extension
-                for ext in blocked_extensions:
-                    await context.route(f"**/*.{ext}", lambda route: route.abort())
-                    
-            return context
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Error creating browser context: {str(e)}", tag="BROWSER")
-            # Fallback to basic context creation if the advanced settings fail
-            return await self.browser.new_context()
-        
-    async def setup_context(self, context: BrowserContext, crawlerRunConfig: Optional[CrawlerRunConfig] = None):
-        """Set up a browser context with the configured options.
-        
-        Args:
-            context: The browser context to set up
-            crawlerRunConfig: Configuration object containing all browser settings
-        """
-        # Set HTTP headers
-        if self.config.headers:
-            await context.set_extra_http_headers(self.config.headers)
-
-        # Add cookies
-        if self.config.cookies:
-            await context.add_cookies(self.config.cookies)
-
-        # Apply storage state if provided
-        if self.config.storage_state:
-            await context.storage_state(path=None)
-
-        # Configure downloads
-        if self.config.accept_downloads:
-            context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT)
-            context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT)
-            if self.config.downloads_path:
-                context._impl_obj._options["accept_downloads"] = True
-                context._impl_obj._options["downloads_path"] = self.config.downloads_path
-
-        # Handle user agent and browser hints
-        if self.config.user_agent:
-            combined_headers = {
-                "User-Agent": self.config.user_agent,
-                "sec-ch-ua": self.config.browser_hint,
-            }
-            combined_headers.update(self.config.headers)
-            await context.set_extra_http_headers(combined_headers)
-
-        # Add default cookie
-        target_url = (crawlerRunConfig and crawlerRunConfig.url) or "https://crawl4ai.com/"
-        await context.add_cookies(
-            [
-                {
-                    "name": "cookiesEnabled",
-                    "value": "true",
-                    "url": target_url,
-                }
-            ]
-        )
-
-        # Handle navigator overrides
-        if crawlerRunConfig:
-            if (
-                crawlerRunConfig.override_navigator
-                or crawlerRunConfig.simulate_user
-                or crawlerRunConfig.magic
-            ):
-                await context.add_init_script(load_js_script("navigator_overrider"))
-                
-    async def kill_session(self, session_id: str):
-        """Kill a browser session and clean up resources.
-
-        Args:
-            session_id (str): The session ID to kill.
-        """
-        if session_id not in self.sessions:
-            return
-            
-        context, page, _ = self.sessions[session_id]
-        
-        # Close the page
-        try:
-            await page.close()
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Error closing page for session {session_id}: {str(e)}", tag="BROWSER")
-        
-        # Remove session from tracking
-        del self.sessions[session_id]
-        
-        # Clean up any contexts that no longer have pages
-        await self._cleanup_unused_contexts()
-        
-        if self.logger:
-            self.logger.debug(f"Killed session: {session_id}", tag="BROWSER")
-
-    async def _cleanup_unused_contexts(self):
-        """Clean up contexts that no longer have any pages."""
-        async with self._contexts_lock:
-            # Get all contexts we're managing
-            contexts_to_check = list(self.contexts_by_config.values())
-            
-            for context in contexts_to_check:
-                # Check if the context has any pages left
-                if not context.pages:
-                    # No pages left, we can close this context
-                    config_signature = next((sig for sig, ctx in self.contexts_by_config.items() 
-                                           if ctx == context), None)
-                    if config_signature:
-                        try:
-                            await context.close()
-                            del self.contexts_by_config[config_signature]
-                            if self.logger:
-                                self.logger.debug(f"Closed unused context", tag="BROWSER")
-                        except Exception as e:
-                            if self.logger:
-                                self.logger.error(f"Error closing unused context: {str(e)}", tag="BROWSER")
-    
-    def _cleanup_expired_sessions(self):
-        """Clean up expired sessions based on TTL."""
-        current_time = time.time()
-        expired_sessions = [
-            sid
-            for sid, (_, _, last_used) in self.sessions.items()
-            if current_time - last_used > self.session_ttl
-        ]
-        
-        for sid in expired_sessions:
-            if self.logger:
-                self.logger.debug(f"Session expired: {sid}", tag="BROWSER")
-            asyncio.create_task(self.kill_session(sid))
-
-    async def close(self):
-        """Close the browser and clean up resources.
-        
-        This method handles common cleanup tasks like:
-        1. Persisting storage state if a user_data_dir is configured
-        2. Closing all sessions
-        3. Closing all browser contexts
-        4. Closing the browser
-        5. Stopping Playwright
-        
-        Child classes should override this method to add their specific cleanup logic,
-        but should call super().close() to ensure common cleanup tasks are performed.
-        """
-        # Set a flag to prevent race conditions during cleanup
-        self.shutting_down = True
-        
-        try:
-            # Add brief delay if configured
-            if self.config.sleep_on_close:
-                await asyncio.sleep(0.5)
-                
-            # Persist storage state if using a user data directory
-            if self.config.user_data_dir and self.browser:
-                for context in self.browser.contexts:
-                    try:
-                        # Ensure the directory exists
-                        storage_dir = os.path.join(self.config.user_data_dir, "Default")
-                        os.makedirs(storage_dir, exist_ok=True)
-                        
-                        # Save storage state
-                        storage_path = os.path.join(storage_dir, "storage_state.json")
-                        await context.storage_state(path=storage_path)
-                        
-                        if self.logger:
-                            self.logger.debug("Storage state persisted before closing browser", tag="BROWSER")
-                    except Exception as e:
-                        if self.logger:
-                            self.logger.warning(
-                                message="Failed to ensure storage persistence: {error}",
-                                tag="BROWSER", 
-                                params={"error": str(e)}
-                            )
-            
-            # Close all active sessions
-            session_ids = list(self.sessions.keys())
-            for session_id in session_ids:
-                await self.kill_session(session_id)
-                
-            # Close all cached contexts
-            for ctx in self.contexts_by_config.values():
-                try:
-                    await ctx.close()
-                except Exception as e:
-                    if self.logger:
-                        self.logger.error(
-                            message="Error closing context: {error}",
-                            tag="BROWSER",
-                            params={"error": str(e)}
-                        )
-            self.contexts_by_config.clear()
-            
-            # Close the browser if it exists
-            if self.browser:
-                await self.browser.close()
-                self.browser = None
-                
-            # Stop playwright
-            if self.playwright:
-                await self.playwright.stop()
-                self.playwright = None
-                
-        except Exception as e:
-            if self.logger:
-                self.logger.error(
-                    message="Error during browser cleanup: {error}",
-                    tag="BROWSER",
-                    params={"error": str(e)}
-                )
-        finally:
-            # Reset shutting down flag
-            self.shutting_down = False
-    
-    

crawl4ai/browser/strategies/builtin.py

@@ -1,468 +0,0 @@
-import asyncio
-import os
-import time
-import json
-import subprocess
-import shutil
-import signal
-from typing import Optional, Dict, Any, Tuple
-
-
-from ...async_logger import AsyncLogger
-from ...async_configs import CrawlerRunConfig
-from playwright.async_api import Page, BrowserContext
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig
-from ...utils import get_home_folder
-from ..utils import get_browser_executable, is_windows, is_browser_running, find_process_by_port, terminate_process
-
-
-from .cdp import CDPBrowserStrategy
-from .base import BaseBrowserStrategy
-
-class BuiltinBrowserStrategy(CDPBrowserStrategy):
-    """Built-in browser strategy.
-    
-    This strategy extends the CDP strategy to use the built-in browser.
-    """
-    
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the built-in browser strategy.
-        
-        Args:
-            config: Browser configuration
-            logger: Logger for recording events and errors
-        """
-        super().__init__(config, logger)
-        self.builtin_browser_dir = os.path.join(get_home_folder(), "builtin-browser") if not self.config.user_data_dir else self.config.user_data_dir
-        self.builtin_config_file = os.path.join(self.builtin_browser_dir, "browser_config.json")
-
-        # Raise error if user data dir is already engaged
-        if self._check_user_dir_is_engaged(self.builtin_browser_dir):
-            raise Exception(f"User data directory {self.builtin_browser_dir} is already engaged by another browser instance.")
-
-        os.makedirs(self.builtin_browser_dir, exist_ok=True)
-    
-    def _check_user_dir_is_engaged(self, user_data_dir: str) -> bool:
-        """Check if the user data directory is already in use.
-        
-        Returns:
-            bool: True if the directory is engaged, False otherwise
-        """
-        # Load browser config file, then iterate in port_map values, check "user_data_dir" key if it matches
-        # the current user data directory
-        if os.path.exists(self.builtin_config_file):
-            try:
-                with open(self.builtin_config_file, 'r') as f:
-                    browser_info_dict = json.load(f)
-                
-                # Check if user data dir is already engaged
-                for port_str, browser_info in browser_info_dict.get("port_map", {}).items():
-                    if browser_info.get("user_data_dir") == user_data_dir:
-                        return True
-            except Exception as e:
-                if self.logger:
-                    self.logger.error(f"Error reading built-in browser config: {str(e)}", tag="BUILTIN")
-        return False
-
-    async def start(self):
-        """Start or connect to the built-in browser.
-        
-        Returns:
-            self: For method chaining
-        """
-        # Initialize Playwright instance via base class method
-        await BaseBrowserStrategy.start(self)
-        
-        try:
-            # Check for existing built-in browser (get_browser_info already checks if running)
-            browser_info = self.get_browser_info()
-            if browser_info:
-                if self.logger:
-                    self.logger.info(f"Using existing built-in browser at {browser_info.get('cdp_url')}", tag="BROWSER")
-                self.config.cdp_url = browser_info.get('cdp_url')
-            else:
-                if self.logger:
-                    self.logger.info("Built-in browser not found, launching new instance...", tag="BROWSER")
-                cdp_url = await self.launch_builtin_browser(
-                    browser_type=self.config.browser_type,
-                    debugging_port=self.config.debugging_port,
-                    headless=self.config.headless,
-                )
-                if not cdp_url:
-                    if self.logger:
-                        self.logger.warning("Failed to launch built-in browser, falling back to regular CDP strategy", tag="BROWSER")
-                    # Call CDP's start but skip BaseBrowserStrategy.start() since we already called it
-                    return await CDPBrowserStrategy.start(self)
-                self.config.cdp_url = cdp_url
-            
-            # Connect to the browser using CDP protocol
-            self.browser = await self.playwright.chromium.connect_over_cdp(self.config.cdp_url)
-            
-            # Get or create default context
-            contexts = self.browser.contexts
-            if contexts:
-                self.default_context = contexts[0]
-            else:
-                self.default_context = await self.create_browser_context()
-            
-            await self.setup_context(self.default_context)
-            
-            if self.logger:
-                self.logger.debug(f"Connected to built-in browser at {self.config.cdp_url}", tag="BUILTIN")
-                
-            return self
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Failed to start built-in browser: {str(e)}", tag="BUILTIN")
-            
-            # There is a possibility that at this point I need to clean up some resourece
-            raise
-
-    def _get_builtin_browser_info(cls, debugging_port: int, config_file: str, logger: Optional[AsyncLogger] = None) -> Optional[Dict[str, Any]]:
-        """Get information about the built-in browser for a specific debugging port.
-        
-        Args:
-            debugging_port: The debugging port to look for
-            config_file: Path to the config file
-            logger: Optional logger for recording events
-            
-        Returns:
-            dict: Browser information or None if no running browser is configured for this port
-        """
-        if not os.path.exists(config_file):
-            return None
-            
-        try:
-            with open(config_file, 'r') as f:
-                browser_info_dict = json.load(f)
-            
-            # Get browser info from port map
-            if isinstance(browser_info_dict, dict) and "port_map" in browser_info_dict:
-                port_str = str(debugging_port)
-                if port_str in browser_info_dict["port_map"]:
-                    browser_info = browser_info_dict["port_map"][port_str]
-                    
-                    # Check if the browser is still running
-                    pids = browser_info.get('pid', '')
-                    if isinstance(pids, str):
-                        pids = [int(pid) for pid in pids.split() if pid.isdigit()]
-                    elif isinstance(pids, int):
-                        pids = [pids]
-                    else:
-                        pids = []
-
-                    # Check if any of the PIDs are running
-                    if not pids:
-                        if logger:
-                            logger.warning(f"Built-in browser on port {debugging_port} has no valid PID", tag="BUILTIN")
-                        # Remove this port from the dictionary
-                        del browser_info_dict["port_map"][port_str]
-                        with open(config_file, 'w') as f:
-                            json.dump(browser_info_dict, f, indent=2)
-                        return None
-                    # Check if any of the PIDs are running
-                    for pid in pids:
-                        if is_browser_running(pid):
-                            browser_info['pid'] = pid
-                            break
-                    else:
-                        # If none of the PIDs are running, remove this port from the dictionary
-                        if logger:
-                            logger.warning(f"Built-in browser on port {debugging_port} is not running", tag="BUILTIN")
-                        # Remove this port from the dictionary
-                        del browser_info_dict["port_map"][port_str]
-                        with open(config_file, 'w') as f:
-                            json.dump(browser_info_dict, f, indent=2)
-                        return None
-                    
-                    return browser_info
-            
-            return None
-                
-        except Exception as e:
-            if logger:
-                logger.error(f"Error reading built-in browser config: {str(e)}", tag="BUILTIN")
-            return None
-            
-    def get_browser_info(self) -> Optional[Dict[str, Any]]:
-        """Get information about the current built-in browser instance.
-        
-        Returns:
-            dict: Browser information or None if no running browser is configured
-        """
-        return self._get_builtin_browser_info(
-            debugging_port=self.config.debugging_port,
-            config_file=self.builtin_config_file,
-            logger=self.logger
-        )
-    
-    async def launch_builtin_browser(self, 
-                               browser_type: str = "chromium",
-                               debugging_port: int = 9222,
-                               headless: bool = True) -> Optional[str]:
-        """Launch a browser in the background for use as the built-in browser.
-        
-        Args:
-            browser_type: Type of browser to launch ('chromium' or 'firefox')
-            debugging_port: Port to use for CDP debugging
-            headless: Whether to run in headless mode
-            
-        Returns:
-            str: CDP URL for the browser, or None if launch failed
-        """
-        # Check if there's an existing browser still running
-        browser_info = self._get_builtin_browser_info(
-            debugging_port=debugging_port,
-            config_file=self.builtin_config_file,
-            logger=self.logger
-        )
-        if browser_info:
-            if self.logger:
-                self.logger.info(f"Built-in browser is already running on port {debugging_port}", tag="BUILTIN")
-            return browser_info.get('cdp_url')
-        
-        # Create a user data directory for the built-in browser
-        user_data_dir = os.path.join(self.builtin_browser_dir, "user_data")
-     
-        # Raise error if user data dir is already engaged
-        if self._check_user_dir_is_engaged(user_data_dir):
-            raise Exception(f"User data directory {user_data_dir} is already engaged by another browser instance.")
-            
-        # Create the user data directory if it doesn't exist
-        os.makedirs(user_data_dir, exist_ok=True)
-        
-        # Prepare browser launch arguments
-        browser_args = super()._build_browser_args()
-        browser_path = await get_browser_executable(browser_type)
-        base_args = [browser_path]
-
-        if browser_type == "chromium":
-            args = [
-                browser_path,
-                f"--remote-debugging-port={debugging_port}",
-                f"--user-data-dir={user_data_dir}",
-            ]
-            # if headless:
-            #     args.append("--headless=new")
-
-        elif browser_type == "firefox":
-            args = [
-                browser_path,
-                "--remote-debugging-port",
-                str(debugging_port),
-                "--profile",
-                user_data_dir,
-            ]
-            if headless:
-                args.append("--headless")
-        else:
-            if self.logger:
-                self.logger.error(f"Browser type {browser_type} not supported for built-in browser", tag="BUILTIN")
-            return None
-        
-        args = base_args + browser_args + args
-        
-        try:
-
-            # Check if the port is already in use
-            PID = ""
-            cdp_url = f"http://localhost:{debugging_port}"
-            config_json = await self._check_port_in_use(cdp_url)
-            if config_json:
-                if self.logger:
-                    self.logger.info(f"Port {debugging_port} is already in use.", tag="BUILTIN")
-                PID = find_process_by_port(debugging_port)
-            else:
-                # Start the browser process detached
-                process = None
-                if is_windows():
-                    process = subprocess.Popen(
-                        args, 
-                        stdout=subprocess.PIPE, 
-                        stderr=subprocess.PIPE,
-                        creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
-                    )
-                else:
-                    process = subprocess.Popen(
-                        args, 
-                        stdout=subprocess.PIPE, 
-                        stderr=subprocess.PIPE,
-                        preexec_fn=os.setpgrp  # Start in a new process group
-                    )
-                
-                # Wait briefly to ensure the process starts successfully
-                await asyncio.sleep(2.0)
-                
-                # Check if the process is still running
-                if process and process.poll() is not None:
-                    if self.logger:
-                        self.logger.error(f"Browser process exited immediately with code {process.returncode}", tag="BUILTIN")
-                    return None
-            
-                PID = process.pid
-                # Construct CDP URL
-                config_json = await self._check_port_in_use(cdp_url)
-
-            
-            # Create browser info
-            browser_info = {
-                'pid': PID,
-                'cdp_url': cdp_url,
-                'user_data_dir': user_data_dir,
-                'browser_type': browser_type,
-                'debugging_port': debugging_port,
-                'start_time': time.time(),
-                'config': config_json
-            }
-            
-            # Read existing config file if it exists
-            port_map = {}
-            if os.path.exists(self.builtin_config_file):
-                try:
-                    with open(self.builtin_config_file, 'r') as f:
-                        existing_data = json.load(f)
-                    
-                    # Check if it already uses port mapping
-                    if isinstance(existing_data, dict) and "port_map" in existing_data:
-                        port_map = existing_data["port_map"]
-
-                    # # Convert legacy format to port mapping
-                    # elif isinstance(existing_data, dict) and "debugging_port" in existing_data:
-                    #     old_port = str(existing_data.get("debugging_port"))
-                    #     if self._is_browser_running(existing_data.get("pid")):
-                    #         port_map[old_port] = existing_data
-                except Exception as e:
-                    if self.logger:
-                        self.logger.warning(f"Could not read existing config: {str(e)}", tag="BUILTIN")
-            
-            # Add/update this browser in the port map
-            port_map[str(debugging_port)] = browser_info
-            
-            # Write updated config
-            with open(self.builtin_config_file, 'w') as f:
-                json.dump({"port_map": port_map}, f, indent=2)
-                
-            # Detach from the browser process - don't keep any references
-            # This is important to allow the Python script to exit while the browser continues running
-            process = None
-                
-            if self.logger:
-                self.logger.success(f"Built-in browser launched at CDP URL: {cdp_url}", tag="BUILTIN")
-            return cdp_url
-            
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Error launching built-in browser: {str(e)}", tag="BUILTIN")
-            return None
-
-    async def _check_port_in_use(self, cdp_url: str) -> dict:
-        """Check if a port is already in use by a Chrome DevTools instance.
-        
-        Args:
-            cdp_url: The CDP URL to check
-            
-        Returns:
-            dict: Chrome DevTools protocol version information or None if not found
-        """
-        import aiohttp
-        json_url = f"{cdp_url}/json/version"
-        json_config = None
-        
-        try:
-            async with aiohttp.ClientSession() as session:
-                try:
-                    async with session.get(json_url, timeout=2.0) as response:
-                        if response.status == 200:
-                            json_config = await response.json()
-                            if self.logger:
-                                self.logger.debug(f"Found CDP server running at {cdp_url}", tag="BUILTIN")
-                            return json_config
-                except (aiohttp.ClientError, asyncio.TimeoutError):
-                    pass
-            return None
-        except Exception as e:
-            if self.logger:
-                self.logger.debug(f"Error checking CDP port: {str(e)}", tag="BUILTIN")
-            return None
-
-    async def kill_builtin_browser(self) -> bool:
-        """Kill the built-in browser if it's running.
-        
-        Returns:
-            bool: True if the browser was killed, False otherwise
-        """
-        browser_info = self.get_browser_info()
-        if not browser_info:
-            if self.logger:
-                self.logger.warning(f"No built-in browser found on port {self.config.debugging_port}", tag="BUILTIN")
-            return False
-            
-        pid = browser_info.get('pid')
-        if not pid:
-            return False
-            
-        success, error_msg = terminate_process(pid, logger=self.logger)
-        if success:
-            # Update config file to remove this browser
-            with open(self.builtin_config_file, 'r') as f:
-                browser_info_dict = json.load(f)
-
-            # Remove this port from the dictionary
-            port_str = str(self.config.debugging_port)
-            if port_str in browser_info_dict.get("port_map", {}):
-                del browser_info_dict["port_map"][port_str]
-
-            with open(self.builtin_config_file, 'w') as f:
-                json.dump(browser_info_dict, f, indent=2)
-
-            # Remove user data directory if it exists
-            if os.path.exists(self.builtin_browser_dir):
-                shutil.rmtree(self.builtin_browser_dir)
-
-            # Clear the browser info cache
-            self.browser = None
-            self.temp_dir = None
-            self.shutting_down = True
-                
-            if self.logger:
-                self.logger.success("Built-in browser terminated", tag="BUILTIN")
-            return True
-        else:
-            if self.logger:
-                self.logger.error(f"Error killing built-in browser: {error_msg}", tag="BUILTIN")
-            return False
-    
-    async def get_builtin_browser_status(self) -> Dict[str, Any]:
-        """Get status information about the built-in browser.
-        
-        Returns:
-            dict: Status information with running, cdp_url, and info fields
-        """
-        browser_info = self.get_browser_info()
-        
-        if not browser_info:
-            return {
-                'running': False,
-                'cdp_url': None,
-                'info': None,
-                'port': self.config.debugging_port
-            }
-            
-        return {
-            'running': True,
-            'cdp_url': browser_info.get('cdp_url'),
-            'info': browser_info,
-            'port': self.config.debugging_port
-        }
-
-    async def close(self):
-        """Close the built-in browser and clean up resources."""
-        # Call parent class close method
-        await super().close()
-        
-        # Clean up built-in browser if we created it and were in shutdown mode
-        if self.shutting_down:
-            await self.kill_builtin_browser()
-            if self.logger:
-                self.logger.debug("Killed built-in browser during shutdown", tag="BUILTIN")

crawl4ai/browser/strategies/cdp.py

@@ -1,281 +0,0 @@
-"""Browser strategies module for Crawl4AI.
-
-This module implements the browser strategy pattern for different
-browser implementations, including Playwright, CDP, and builtin browsers.
-"""
-
-import asyncio
-import os
-import time
-import json
-import subprocess
-import shutil
-from typing import Optional, Tuple, List
-
-from playwright.async_api import BrowserContext, Page
-
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig, CrawlerRunConfig
-from ..utils import get_playwright, get_browser_executable, create_temp_directory, is_windows, check_process_is_running, terminate_process
-
-from .base import BaseBrowserStrategy
-
-class CDPBrowserStrategy(BaseBrowserStrategy):
-    """CDP-based browser strategy.
-    
-    This strategy connects to an existing browser using CDP protocol or
-    launches and connects to a browser using CDP.
-    """
-    
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the CDP browser strategy.
-        
-        Args:
-            config: Browser configuration
-            logger: Logger for recording events and errors
-        """
-        super().__init__(config, logger)
-        self.sessions = {}
-        self.session_ttl = 1800  # 30 minutes
-        self.browser_process = None
-        self.temp_dir = None
-        self.shutting_down = False
-        
-    async def start(self):
-        """Start or connect to the browser using CDP.
-        
-        Returns:
-            self: For method chaining
-        """
-        # Call the base class start to initialize Playwright
-        await super().start()
-        
-        try:
-            # Get or create CDP URL
-            cdp_url = await self._get_or_create_cdp_url()
-            
-            # Connect to the browser using CDP
-            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
-            
-            # Get or create default context
-            contexts = self.browser.contexts
-            if contexts:
-                self.default_context = contexts[0]
-            else:
-                self.default_context = await self.create_browser_context()
-            
-            await self.setup_context(self.default_context)
-            
-            if self.logger:
-                self.logger.debug(f"Connected to CDP browser at {cdp_url}", tag="CDP")
-
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Failed to connect to CDP browser: {str(e)}", tag="CDP")
-
-            # Clean up any resources before re-raising
-            await self._cleanup_process()
-            raise
-            
-        return self
-    
-    async def _get_or_create_cdp_url(self) -> str:
-        """Get existing CDP URL or launch a browser and return its CDP URL.
-        
-        Returns:
-            str: CDP URL for connecting to the browser
-        """
-        # If CDP URL is provided, just return it
-        if self.config.cdp_url:
-            return self.config.cdp_url
-
-        # Create temp dir if needed
-        if not self.config.user_data_dir:
-            self.temp_dir = create_temp_directory()
-            user_data_dir = self.temp_dir
-        else:
-            user_data_dir = self.config.user_data_dir
-
-        # Get browser args based on OS and browser type
-        # args = await self._get_browser_args(user_data_dir)
-        browser_args = super()._build_browser_args()
-        browser_path = await get_browser_executable(self.config.browser_type)
-        base_args = [browser_path]
-
-        if self.config.browser_type == "chromium":
-            args = [
-                f"--remote-debugging-port={self.config.debugging_port}",
-                f"--user-data-dir={user_data_dir}",
-            ]
-            # if self.config.headless:
-            #     args.append("--headless=new")
-
-        elif self.config.browser_type == "firefox":
-            args = [
-                "--remote-debugging-port",
-                str(self.config.debugging_port),
-                "--profile",
-                user_data_dir,
-            ]
-            if self.config.headless:
-                args.append("--headless")
-        else:
-            raise NotImplementedError(f"Browser type {self.config.browser_type} not supported")
-
-        args = base_args + browser_args['args'] + args         
-
-        # Start browser process
-        try:
-            # Use DETACHED_PROCESS flag on Windows to fully detach the process
-            # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group
-            if is_windows():
-                self.browser_process = subprocess.Popen(
-                    args, 
-                    stdout=subprocess.PIPE, 
-                    stderr=subprocess.PIPE,
-                    creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
-                )
-            else:
-                self.browser_process = subprocess.Popen(
-                    args, 
-                    stdout=subprocess.PIPE, 
-                    stderr=subprocess.PIPE,
-                    preexec_fn=os.setpgrp  # Start in a new process group
-                )
-                
-            # Monitor for a short time to make sure it starts properly
-            is_running, return_code, stdout, stderr = await check_process_is_running(self.browser_process, delay=2)
-            if not is_running:
-                if self.logger:
-                    self.logger.error(
-                        message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
-                        tag="ERROR",
-                        params={
-                            "code": return_code,
-                            "stdout": stdout.decode() if stdout else "",
-                            "stderr": stderr.decode() if stderr else "",
-                        },
-                    )
-                await self._cleanup_process()
-                raise Exception("Browser process terminated unexpectedly")
-
-            return f"http://localhost:{self.config.debugging_port}"
-        except Exception as e:
-            await self._cleanup_process()
-            raise Exception(f"Failed to start browser: {e}")    
-
-    async def _cleanup_process(self):
-        """Cleanup browser process and temporary directory."""
-        # Set shutting_down flag BEFORE any termination actions
-        self.shutting_down = True
-
-        if self.browser_process:
-            try:
-                # Only attempt termination if the process is still running
-                if self.browser_process.poll() is None:
-                    # Use our robust cross-platform termination utility
-                    success = terminate_process(
-                        pid=self.browser_process.pid,
-                        timeout=1.0,  # Equivalent to the previous 10*0.1s wait
-                        logger=self.logger
-                    )
-                    
-                    if not success and self.logger:
-                        self.logger.warning(
-                            message="Failed to terminate browser process cleanly",
-                            tag="PROCESS"
-                        )
-                        
-            except Exception as e:
-                if self.logger:
-                    self.logger.error(
-                        message="Error during browser process cleanup: {error}",
-                        tag="ERROR",
-                        params={"error": str(e)},
-                    )
-
-        if self.temp_dir and os.path.exists(self.temp_dir):
-            try:
-                shutil.rmtree(self.temp_dir)
-                self.temp_dir = None
-                if self.logger:
-                    self.logger.debug("Removed temporary directory", tag="CDP")
-            except Exception as e:
-                if self.logger:
-                    self.logger.error(
-                        message="Error removing temporary directory: {error}",
-                        tag="CDP",
-                        params={"error": str(e)}
-                    )
-
-        self.browser_process = None
-    
-    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        # For CDP, we typically use the shared default_context
-        context = self.default_context
-        pages = context.pages
-        
-        # Otherwise, check if we have an existing context for this config
-        config_signature = self._make_config_signature(crawlerRunConfig)
-        self.contexts_by_config[config_signature] = context
-
-        await self.setup_context(context, crawlerRunConfig)
-
-        # Check if there's already a page with the target URL
-        page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
-        
-        # If not found, create a new page
-        if not page:
-            page = await context.new_page()
-        
-        return page, context
-
-    async def _get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        """Get a page for the given configuration.
-        
-        Args:
-            crawlerRunConfig: Configuration object for the crawler run
-            
-        Returns:
-            Tuple of (Page, BrowserContext)
-        """
-        # Call parent method to ensure browser is started
-        await super().get_page(crawlerRunConfig)
-        
-        # For CDP, we typically use the shared default_context
-        context = self.default_context
-        pages = context.pages
-        
-        # Otherwise, check if we have an existing context for this config
-        config_signature = self._make_config_signature(crawlerRunConfig)
-        self.contexts_by_config[config_signature] = context
-
-        await self.setup_context(context, crawlerRunConfig)
-
-        # Check if there's already a page with the target URL
-        page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
-        
-        # If not found, create a new page
-        if not page:
-            page = await context.new_page()
-        
-        # If a session_id is specified, store this session for reuse
-        if crawlerRunConfig.session_id:
-            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
-        
-        return page, context
-
-    async def close(self):
-        """Close the CDP browser and clean up resources."""
-        # Skip cleanup if using external CDP URL and not launched by us
-        if self.config.cdp_url and not self.browser_process:
-            if self.logger:
-                self.logger.debug("Skipping cleanup for external CDP browser", tag="CDP")
-            return
-        
-        # Call parent implementation for common cleanup
-        await super().close()
-        
-        # Additional CDP-specific cleanup
-        await asyncio.sleep(0.5)
-        await self._cleanup_process()

crawl4ai/browser/strategies/docker_strategy.py

@@ -1,430 +0,0 @@
-"""Docker browser strategy module for Crawl4AI.
-
-This module provides browser strategies for running browsers in Docker containers,
-which offers better isolation, consistency across platforms, and easy scaling.
-"""
-
-import os
-import uuid
-from typing import List, Optional
-
-
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig
-from ..models import DockerConfig
-from ..docker_registry import DockerRegistry
-from ..docker_utils import DockerUtils
-from .builtin import CDPBrowserStrategy
-from .base import BaseBrowserStrategy
-
-class DockerBrowserStrategy(CDPBrowserStrategy):
-    """Docker-based browser strategy.
-
-    Extends the CDPBrowserStrategy to run browsers in Docker containers.
-    Supports two modes:
-    1. "connect" - Uses a Docker image with Chrome already running
-    2. "launch" - Starts Chrome within the container with custom settings
-
-    Attributes:
-        docker_config: Docker-specific configuration options
-        container_id: ID of current Docker container
-        container_name: Name assigned to the container
-        registry: Registry for tracking and reusing containers
-        docker_utils: Utilities for Docker operations
-        chrome_process_id: Process ID of Chrome within container
-        socat_process_id: Process ID of socat within container
-        internal_cdp_port: Chrome's internal CDP port
-        internal_mapped_port: Port that socat maps to internally
-    """
-
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the Docker browser strategy.
-
-        Args:
-            config: Browser configuration including Docker-specific settings
-            logger: Logger for recording events and errors
-        """
-        super().__init__(config, logger)
-
-        # Initialize Docker-specific attributes
-        self.docker_config = self.config.docker_config or DockerConfig()
-        self.container_id = None
-        self.container_name = f"crawl4ai-browser-{uuid.uuid4().hex[:8]}"
-
-        # Use the shared registry file path for consistency with BuiltinBrowserStrategy
-        registry_file = self.docker_config.registry_file
-        if registry_file is None and self.config.user_data_dir:
-            # Use the same registry file as BuiltinBrowserStrategy if possible
-            registry_file = os.path.join(
-                os.path.dirname(self.config.user_data_dir), "browser_config.json"
-            )
-
-        self.registry = DockerRegistry(self.docker_config.registry_file)
-        self.docker_utils = DockerUtils(logger)
-        self.chrome_process_id = None
-        self.socat_process_id = None
-        self.internal_cdp_port = 9222  # Chrome's internal CDP port
-        self.internal_mapped_port = 9223  # Port that socat maps to internally
-        self.shutting_down = False
-
-    async def start(self):
-        """Start or connect to a browser running in a Docker container.
-
-        This method initializes Playwright and establishes a connection to
-        a browser running in a Docker container. Depending on the configured mode:
-        - "connect": Connects to a container with Chrome already running
-        - "launch": Creates a container and launches Chrome within it
-
-        Returns:
-            self: For method chaining
-        """
-        # Initialize Playwright
-        await BaseBrowserStrategy.start(self)
-
-        if self.logger:
-            self.logger.info(
-                f"Starting Docker browser strategy in {self.docker_config.mode} mode",
-                tag="DOCKER",
-            )
-
-        try:
-            # Get CDP URL by creating or reusing a Docker container
-            # This handles the container management and browser startup
-            cdp_url = await self._get_or_create_cdp_url()
-
-            if not cdp_url:
-                raise Exception(
-                    "Failed to establish CDP connection to Docker container"
-                )
-
-            if self.logger:
-                self.logger.info(
-                    f"Connecting to browser in Docker via CDP: {cdp_url}", tag="DOCKER"
-                )
-
-            # Connect to the browser using CDP
-            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
-
-            # Get existing context or create default context
-            contexts = self.browser.contexts
-            if contexts:
-                self.default_context = contexts[0]
-                if self.logger:
-                    self.logger.debug("Using existing browser context", tag="DOCKER")
-            else:
-                if self.logger:
-                    self.logger.debug("Creating new browser context", tag="DOCKER")
-                self.default_context = await self.create_browser_context()
-                await self.setup_context(self.default_context)
-
-            return self
-
-        except Exception as e:
-            # Clean up resources if startup fails
-            if self.container_id and not self.docker_config.persistent:
-                if self.logger:
-                    self.logger.warning(
-                        f"Cleaning up container after failed start: {self.container_id[:12]}",
-                        tag="DOCKER",
-                    )
-                await self.docker_utils.remove_container(self.container_id)
-                self.registry.unregister_container(self.container_id)
-                self.container_id = None
-
-            if self.playwright:
-                await self.playwright.stop()
-                self.playwright = None
-
-            # Re-raise the exception
-            if self.logger:
-                self.logger.error(
-                    f"Failed to start Docker browser: {str(e)}", tag="DOCKER"
-                )
-            raise
-
-    async def _generate_config_hash(self) -> str:
-        """Generate a hash of the configuration for container matching.
-
-        Returns:
-            Hash string uniquely identifying this configuration
-        """
-        # Create a dict with the relevant parts of the config
-        config_dict = {
-            "image": self.docker_config.image,
-            "mode": self.docker_config.mode,
-            "browser_type": self.config.browser_type,
-            "headless": self.config.headless,
-        }
-
-        # Add browser-specific config if in launch mode
-        if self.docker_config.mode == "launch":
-            config_dict.update(
-                {
-                    "text_mode": self.config.text_mode,
-                    "light_mode": self.config.light_mode,
-                    "viewport_width": self.config.viewport_width,
-                    "viewport_height": self.config.viewport_height,
-                }
-            )
-
-        # Use the utility method to generate the hash
-        return self.docker_utils.generate_config_hash(config_dict)
-
-    async def _get_or_create_cdp_url(self) -> str:
-        """Get CDP URL by either creating a new container or using an existing one.
-
-        Returns:
-            CDP URL for connecting to the browser
-
-        Raises:
-            Exception: If container creation or browser launch fails
-        """
-        # If CDP URL is explicitly provided, use it
-        if self.config.cdp_url:
-            return self.config.cdp_url
-
-        # Ensure Docker image exists (will build if needed)
-        image_name = await self.docker_utils.ensure_docker_image_exists(
-            self.docker_config.image, self.docker_config.mode
-        )
-
-        # Generate config hash for container matching
-        config_hash = await self._generate_config_hash()
-
-        # Look for existing container with matching config
-        container_id = await self.registry.find_container_by_config(
-            config_hash, self.docker_utils
-        )
-
-        if container_id:
-            # Use existing container
-            self.container_id = container_id
-            host_port = self.registry.get_container_host_port(container_id)
-            if self.logger:
-                self.logger.info(
-                    f"Using existing Docker container: {container_id[:12]}",
-                    tag="DOCKER",
-                )
-        else:
-            # Get a port for the new container
-            host_port = (
-                self.docker_config.host_port
-                or self.registry.get_next_available_port(self.docker_utils)
-            )
-
-            # Prepare volumes list
-            volumes = list(self.docker_config.volumes)
-
-            # Add user data directory if specified
-            if self.docker_config.user_data_dir:
-                # Ensure user data directory exists
-                os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
-                volumes.append(
-                    f"{self.docker_config.user_data_dir}:{self.docker_config.container_user_data_dir}"
-                )
-
-                # # Update config user_data_dir to point to container path
-                # self.config.user_data_dir = self.docker_config.container_user_data_dir
-
-            # Create a new container
-            container_id = await self.docker_utils.create_container(
-                image_name=image_name,
-                host_port=host_port,
-                container_name=self.container_name,
-                volumes=volumes,
-                network=self.docker_config.network,
-                env_vars=self.docker_config.env_vars,
-                cpu_limit=self.docker_config.cpu_limit,
-                memory_limit=self.docker_config.memory_limit,
-                extra_args=self.docker_config.extra_args,
-            )
-
-            if not container_id:
-                raise Exception("Failed to create Docker container")
-
-            self.container_id = container_id
-
-            # Wait for container to be ready
-            await self.docker_utils.wait_for_container_ready(container_id)
-
-            # Handle specific setup based on mode
-            if self.docker_config.mode == "launch":
-                # In launch mode, we need to start socat and Chrome
-                await self.docker_utils.start_socat_in_container(container_id)
-
-                # Build browser arguments
-                browser_args = self._build_browser_args()
-
-                # Launch Chrome
-                await self.docker_utils.launch_chrome_in_container(
-                    container_id, browser_args
-                )
-
-                # Get PIDs for later cleanup
-                self.chrome_process_id = (
-                    await self.docker_utils.get_process_id_in_container(
-                        container_id, "chromium"
-                    )
-                )
-                self.socat_process_id = (
-                    await self.docker_utils.get_process_id_in_container(
-                        container_id, "socat"
-                    )
-                )
-
-            # Wait for CDP to be ready
-            cdp_json_config = await self.docker_utils.wait_for_cdp_ready(host_port)
-
-            if cdp_json_config:
-                # Register the container in the shared registry
-                self.registry.register_container(
-                    container_id, host_port, config_hash, cdp_json_config
-                )
-            else:
-                raise Exception("Failed to get CDP JSON config from Docker container")
-
-            if self.logger:
-                self.logger.success(
-                    f"Docker container ready: {container_id[:12]} on port {host_port}",
-                    tag="DOCKER",
-                )
-
-        # Return CDP URL
-        return f"http://localhost:{host_port}"
-
-    def _build_browser_args(self) -> List[str]:
-        """Build Chrome command line arguments based on BrowserConfig.
-
-        Returns:
-            List of command line arguments for Chrome
-        """
-        # Call parent method to get common arguments
-        browser_args = super()._build_browser_args()
-        return browser_args["args"] + [
-            f"--remote-debugging-port={self.internal_cdp_port}",
-            "--remote-debugging-address=0.0.0.0",  # Allow external connections
-            "--disable-dev-shm-usage",
-            "--headless=new",
-        ]
-
-        # args = [
-        #     "--no-sandbox",
-        #     "--disable-gpu",
-        #     f"--remote-debugging-port={self.internal_cdp_port}",
-        #     "--remote-debugging-address=0.0.0.0",  # Allow external connections
-        #     "--disable-dev-shm-usage",
-        # ]
-
-        # if self.config.headless:
-        #     args.append("--headless=new")
-
-        # if self.config.viewport_width and self.config.viewport_height:
-        #     args.append(f"--window-size={self.config.viewport_width},{self.config.viewport_height}")
-
-        # if self.config.user_agent:
-        #     args.append(f"--user-agent={self.config.user_agent}")
-
-        # if self.config.text_mode:
-        #     args.extend([
-        #         "--blink-settings=imagesEnabled=false",
-        #         "--disable-remote-fonts",
-        #         "--disable-images",
-        #         "--disable-javascript",
-        #     ])
-
-        # if self.config.light_mode:
-        #     # Import here to avoid circular import
-        #     from ..utils import get_browser_disable_options
-        #     args.extend(get_browser_disable_options())
-
-        # if self.config.user_data_dir:
-        #     args.append(f"--user-data-dir={self.config.user_data_dir}")
-
-        # if self.config.extra_args:
-        #     args.extend(self.config.extra_args)
-
-        # return args
-
-    async def close(self):
-        """Close the browser and clean up Docker container if needed."""
-        # Set flag to track if we were the ones initiating shutdown
-        initiated_shutdown = not self.shutting_down
-        # Storage persistence for Docker needs special handling
-        # We need to store state before calling super().close() which will close the browser
-        if (
-            self.browser
-            and self.docker_config.user_data_dir
-            and self.docker_config.persistent
-        ):
-            for context in self.browser.contexts:
-                try:
-                    # Ensure directory exists
-                    os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
-
-                    # Save storage state to user data directory
-                    storage_path = os.path.join(
-                        self.docker_config.user_data_dir, "storage_state.json"
-                    )
-                    await context.storage_state(path=storage_path)
-                    if self.logger:
-                        self.logger.debug(
-                            "Persisted Docker-specific storage state", tag="DOCKER"
-                        )
-                except Exception as e:
-                    if self.logger:
-                        self.logger.warning(
-                            message="Failed to persist Docker storage state: {error}",
-                            tag="DOCKER",
-                            params={"error": str(e)},
-                        )
-
-        # Call parent method to handle common cleanup
-        await super().close()
-
-        # Only perform container cleanup if we initiated shutdown
-        # and we need to handle Docker-specific resources
-        if initiated_shutdown:
-            # Only clean up container if not persistent
-            if self.container_id and not self.docker_config.persistent:
-                # Stop Chrome process in "launch" mode
-                if self.docker_config.mode == "launch" and self.chrome_process_id:
-                    await self.docker_utils.stop_process_in_container(
-                        self.container_id, self.chrome_process_id
-                    )
-                    if self.logger:
-                        self.logger.debug(
-                            f"Stopped Chrome process {self.chrome_process_id} in container",
-                            tag="DOCKER",
-                        )
-
-                # Stop socat process in "launch" mode
-                if self.docker_config.mode == "launch" and self.socat_process_id:
-                    await self.docker_utils.stop_process_in_container(
-                        self.container_id, self.socat_process_id
-                    )
-                    if self.logger:
-                        self.logger.debug(
-                            f"Stopped socat process {self.socat_process_id} in container",
-                            tag="DOCKER",
-                        )
-
-                # Remove or stop container based on configuration
-                if self.docker_config.remove_on_exit:
-                    await self.docker_utils.remove_container(self.container_id)
-                    # Unregister from registry
-                    if hasattr(self, "registry") and self.registry:
-                        self.registry.unregister_container(self.container_id)
-                    if self.logger:
-                        self.logger.debug(
-                            f"Removed Docker container {self.container_id}",
-                            tag="DOCKER",
-                        )
-                else:
-                    await self.docker_utils.stop_container(self.container_id)
-                    if self.logger:
-                        self.logger.debug(
-                            f"Stopped Docker container {self.container_id}",
-                            tag="DOCKER",
-                        )
-
-                self.container_id = None

crawl4ai/browser/strategies/playwright.py

@@ -1,134 +0,0 @@
-"""Browser strategies module for Crawl4AI.
-
-This module implements the browser strategy pattern for different
-browser implementations, including Playwright, CDP, and builtin browsers.
-"""
-
-import time
-from typing import Optional, Tuple
-
-from playwright.async_api import BrowserContext, Page
-
-from ...async_logger import AsyncLogger
-from ...async_configs import BrowserConfig, CrawlerRunConfig
-
-from playwright_stealth import StealthConfig
-
-from .base import BaseBrowserStrategy
-
-stealth_config = StealthConfig(
-    webdriver=True,
-    chrome_app=True,
-    chrome_csi=True,
-    chrome_load_times=True,
-    chrome_runtime=True,
-    navigator_languages=True,
-    navigator_plugins=True,
-    navigator_permissions=True,
-    webgl_vendor=True,
-    outerdimensions=True,
-    navigator_hardware_concurrency=True,
-    media_codecs=True,
-)
-
-class PlaywrightBrowserStrategy(BaseBrowserStrategy):
-    """Standard Playwright browser strategy.
-    
-    This strategy launches a new browser instance using Playwright
-    and manages browser contexts.
-    """
-    
-    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
-        """Initialize the Playwright browser strategy.
-        
-        Args:
-            config: Browser configuration
-            logger: Logger for recording events and errors
-        """
-        super().__init__(config, logger)
-        # No need to re-initialize sessions and session_ttl as they're now in the base class
-    
-    async def start(self):
-        """Start the browser instance.
-        
-        Returns:
-            self: For method chaining
-        """
-        # Call the base class start to initialize Playwright
-        await super().start()
-        
-        # Build browser arguments using the base class method
-        browser_args = self._build_browser_args()
-        
-        try:
-            # Launch appropriate browser type
-            if self.config.browser_type == "firefox":
-                self.browser = await self.playwright.firefox.launch(**browser_args)
-            elif self.config.browser_type == "webkit":
-                self.browser = await self.playwright.webkit.launch(**browser_args)
-            else:
-                self.browser = await self.playwright.chromium.launch(**browser_args)
-                
-            self.default_context = self.browser
-            
-            if self.logger:
-                self.logger.debug(f"Launched {self.config.browser_type} browser", tag="BROWSER")
-                
-        except Exception as e:
-            if self.logger:
-                self.logger.error(f"Failed to launch browser: {str(e)}", tag="BROWSER")
-            raise
-            
-        return self
-
-    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        # Otherwise, check if we have an existing context for this config
-        config_signature = self._make_config_signature(crawlerRunConfig)
-        
-        async with self._contexts_lock:
-            if config_signature in self.contexts_by_config:
-                context = self.contexts_by_config[config_signature]
-            else:
-                # Create and setup a new context
-                context = await self.create_browser_context(crawlerRunConfig)
-                await self.setup_context(context, crawlerRunConfig)
-                self.contexts_by_config[config_signature] = context
-
-        # Create a new page from the chosen context
-        page = await context.new_page()
-        
-        return page, context
-
-    async def _get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
-        """Get a page for the given configuration.
-        
-        Args:
-            crawlerRunConfig: Configuration object for the crawler run
-            
-        Returns:
-            Tuple of (Page, BrowserContext)
-        """
-        # Call parent method to ensure browser is started
-        await super().get_page(crawlerRunConfig)
-        
-        # Otherwise, check if we have an existing context for this config
-        config_signature = self._make_config_signature(crawlerRunConfig)
-        
-        async with self._contexts_lock:
-            if config_signature in self.contexts_by_config:
-                context = self.contexts_by_config[config_signature]
-            else:
-                # Create and setup a new context
-                context = await self.create_browser_context(crawlerRunConfig)
-                await self.setup_context(context, crawlerRunConfig)
-                self.contexts_by_config[config_signature] = context
-
-        # Create a new page from the chosen context
-        page = await context.new_page()
-        
-        # If a session_id is specified, store this session so we can reuse later
-        if crawlerRunConfig.session_id:
-            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
-            
-        return page, context
-        

crawl4ai/browser/utils.py

@@ -1,465 +0,0 @@
-"""Browser utilities module for Crawl4AI.
-
-This module provides utility functions for browser management,
-including process management, CDP connection utilities,
-and Playwright instance management.
-"""
-
-import asyncio
-import os
-import sys
-import time
-import tempfile
-import subprocess
-from typing import Optional, Tuple, Union
-import signal
-import psutil
-
-from playwright.async_api import async_playwright
-
-from ..utils import get_chromium_path
-from ..async_configs import BrowserConfig, CrawlerRunConfig
-
-from ..async_logger import AsyncLogger
-
-
-_playwright_instance = None
-
-async def get_playwright():
-    """Get or create the Playwright instance (singleton pattern).
-    
-    Returns:
-        Playwright: The Playwright instance
-    """
-    global _playwright_instance
-    if _playwright_instance is None or True:
-        _playwright_instance = await async_playwright().start()
-    return _playwright_instance
-
-async def get_browser_executable(browser_type: str) -> str:
-    """Get the path to browser executable, with platform-specific handling.
-    
-    Args:
-        browser_type: Type of browser (chromium, firefox, webkit)
-        
-    Returns:
-        Path to browser executable
-    """
-    return await get_chromium_path(browser_type)
-
-def create_temp_directory(prefix="browser-profile-") -> str:
-    """Create a temporary directory for browser data.
-    
-    Args:
-        prefix: Prefix for the temporary directory name
-        
-    Returns:
-        Path to the created temporary directory
-    """
-    return tempfile.mkdtemp(prefix=prefix)
-
-def is_windows() -> bool:
-    """Check if the current platform is Windows.
-    
-    Returns:
-        True if Windows, False otherwise
-    """
-    return sys.platform == "win32"
-
-def is_macos() -> bool:
-    """Check if the current platform is macOS.
-    
-    Returns:
-        True if macOS, False otherwise
-    """
-    return sys.platform == "darwin"
-
-def is_linux() -> bool:
-    """Check if the current platform is Linux.
-    
-    Returns:
-        True if Linux, False otherwise
-    """
-    return not (is_windows() or is_macos())
-    
-def is_browser_running(pid: Optional[int]) -> bool:
-    """Check if a process with the given PID is running.
-    
-    Args:
-        pid: Process ID to check
-        
-    Returns:
-        bool: True if the process is running, False otherwise
-    """
-    if not pid:
-        return False
-        
-    try:
-        if type(pid) is str:
-            pid = int(pid)
-        # Check if the process exists
-        if is_windows():
-            process = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], 
-                                     capture_output=True, text=True)
-            return str(pid) in process.stdout
-        else:
-            # Unix-like systems
-            os.kill(pid, 0)  # This doesn't actually kill the process, just checks if it exists
-        return True
-    except (ProcessLookupError, PermissionError, OSError):
-        return False
-
-def get_browser_disable_options() -> list:
-    """Get standard list of browser disable options for performance.
-    
-    Returns:
-        List of command-line options to disable various browser features
-    """
-    return [
-        "--disable-background-networking",
-        "--disable-background-timer-throttling",
-        "--disable-backgrounding-occluded-windows",
-        "--disable-breakpad",
-        "--disable-client-side-phishing-detection",
-        "--disable-component-extensions-with-background-pages",
-        "--disable-default-apps",
-        "--disable-extensions",
-        "--disable-features=TranslateUI",
-        "--disable-hang-monitor",
-        "--disable-ipc-flooding-protection",
-        "--disable-popup-blocking",
-        "--disable-prompt-on-repost",
-        "--disable-sync",
-        "--force-color-profile=srgb",
-        "--metrics-recording-only",
-        "--no-first-run",
-        "--password-store=basic",
-        "--use-mock-keychain",
-    ]
-
-
-async def find_optimal_browser_config(total_urls=50, verbose=True, rate_limit_delay=0.2):
-    """Find optimal browser configuration for crawling a specific number of URLs.
-    
-    Args:
-        total_urls: Number of URLs to crawl
-        verbose: Whether to print progress
-        rate_limit_delay: Delay between page loads to avoid rate limiting
-        
-    Returns:
-        dict: Contains fastest, lowest_memory, and optimal configurations
-    """
-    from .manager import BrowserManager
-    if verbose:
-        print(f"\n=== Finding optimal configuration for crawling {total_urls} URLs ===\n")
-    
-    # Generate test URLs with timestamp to avoid caching
-    timestamp = int(time.time())
-    urls = [f"https://example.com/page_{i}?t={timestamp}" for i in range(total_urls)]
-    
-    # Limit browser configurations to test (1 browser to max 10)
-    max_browsers = min(10, total_urls)
-    configs_to_test = []
-    
-    # Generate configurations (browser count, pages distribution)
-    for num_browsers in range(1, max_browsers + 1):
-        base_pages = total_urls // num_browsers
-        remainder = total_urls % num_browsers
-        
-        # Create distribution array like [3, 3, 2, 2] (some browsers get one more page)
-        if remainder > 0:
-            distribution = [base_pages + 1] * remainder + [base_pages] * (num_browsers - remainder)
-        else:
-            distribution = [base_pages] * num_browsers
-            
-        configs_to_test.append((num_browsers, distribution))
-    
-    results = []
-    
-    # Test each configuration
-    for browser_count, page_distribution in configs_to_test:
-        if verbose:
-            print(f"Testing {browser_count} browsers with distribution {tuple(page_distribution)}")
-        
-        try:
-            # Track memory if possible
-            try:
-                import psutil
-                process = psutil.Process()
-                start_memory = process.memory_info().rss / (1024 * 1024)  # MB
-            except ImportError:
-                if verbose: 
-                    print("Memory tracking not available (psutil not installed)")
-                start_memory = 0
-            
-            # Start browsers in parallel
-            managers = []
-            start_tasks = []
-            start_time = time.time()
-
-            logger = AsyncLogger(verbose=True, log_file=None)
-            
-            for i in range(browser_count):
-                config = BrowserConfig(headless=True)
-                manager = BrowserManager(browser_config=config, logger=logger)
-                start_tasks.append(manager.start())
-                managers.append(manager)
-            
-            await asyncio.gather(*start_tasks)
-            
-            # Distribute URLs among browsers
-            urls_per_manager = {}
-            url_index = 0
-            
-            for i, manager in enumerate(managers):
-                pages_for_this_browser = page_distribution[i]
-                end_index = url_index + pages_for_this_browser
-                urls_per_manager[manager] = urls[url_index:end_index]
-                url_index = end_index
-            
-            # Create pages for each browser
-            all_pages = []
-            for manager, manager_urls in urls_per_manager.items():
-                if not manager_urls:
-                    continue
-                pages = await manager.get_pages(CrawlerRunConfig(), count=len(manager_urls))
-                all_pages.extend(zip(pages, manager_urls))
-            
-            # Crawl pages with delay to avoid rate limiting
-            async def crawl_page(page_ctx, url):
-                page, _ = page_ctx
-                try:
-                    await page.goto(url)
-                    if rate_limit_delay > 0:
-                        await asyncio.sleep(rate_limit_delay)
-                    title = await page.title()
-                    return title
-                finally:
-                    await page.close()
-            
-            crawl_start = time.time()
-            crawl_tasks = [crawl_page(page_ctx, url) for page_ctx, url in all_pages]
-            await asyncio.gather(*crawl_tasks)
-            crawl_time = time.time() - crawl_start
-            total_time = time.time() - start_time
-            
-            # Measure final memory usage
-            if start_memory > 0:
-                end_memory = process.memory_info().rss / (1024 * 1024)
-                memory_used = end_memory - start_memory
-            else:
-                memory_used = 0
-            
-            # Close all browsers
-            for manager in managers:
-                await manager.close()
-            
-            # Calculate metrics
-            pages_per_second = total_urls / crawl_time
-            
-            # Calculate efficiency score (higher is better)
-            # This balances speed vs memory
-            if memory_used > 0:
-                efficiency = pages_per_second / (memory_used + 1)
-            else:
-                efficiency = pages_per_second
-            
-            # Store result
-            result = {
-                "browser_count": browser_count,
-                "distribution": tuple(page_distribution),
-                "crawl_time": crawl_time,
-                "total_time": total_time,
-                "memory_used": memory_used,
-                "pages_per_second": pages_per_second, 
-                "efficiency": efficiency
-            }
-            
-            results.append(result)
-            
-            if verbose:
-                print(f"  ✓ Crawled {total_urls} pages in {crawl_time:.2f}s ({pages_per_second:.1f} pages/sec)")
-                if memory_used > 0:
-                    print(f"  ✓ Memory used: {memory_used:.1f}MB ({memory_used/total_urls:.1f}MB per page)")
-                print(f"  ✓ Efficiency score: {efficiency:.4f}")
-            
-        except Exception as e:
-            if verbose:
-                print(f"  ✗ Error: {str(e)}")
-            
-            # Clean up
-            for manager in managers:
-                try:
-                    await manager.close()
-                except:
-                    pass
-    
-    # If no successful results, return None
-    if not results:
-        return None
-    
-    # Find best configurations
-    fastest = sorted(results, key=lambda x: x["crawl_time"])[0]
-    
-    # Only consider memory if available
-    memory_results = [r for r in results if r["memory_used"] > 0]
-    if memory_results:
-        lowest_memory = sorted(memory_results, key=lambda x: x["memory_used"])[0]
-    else:
-        lowest_memory = fastest
-    
-    # Find most efficient (balanced speed vs memory)
-    optimal = sorted(results, key=lambda x: x["efficiency"], reverse=True)[0]
-    
-    # Print summary
-    if verbose:
-        print("\n=== OPTIMAL CONFIGURATIONS ===")
-        print(f"⚡ Fastest: {fastest['browser_count']} browsers {fastest['distribution']}")
-        print(f"   {fastest['crawl_time']:.2f}s, {fastest['pages_per_second']:.1f} pages/sec")
-        
-        print(f"💾 Memory-efficient: {lowest_memory['browser_count']} browsers {lowest_memory['distribution']}")
-        if lowest_memory["memory_used"] > 0:
-            print(f"   {lowest_memory['memory_used']:.1f}MB, {lowest_memory['memory_used']/total_urls:.2f}MB per page")
-        
-        print(f"🌟 Balanced optimal: {optimal['browser_count']} browsers {optimal['distribution']}")
-        print(f"   {optimal['crawl_time']:.2f}s, {optimal['pages_per_second']:.1f} pages/sec, score: {optimal['efficiency']:.4f}")
-    
-    return {
-        "fastest": fastest,
-        "lowest_memory": lowest_memory,
-        "optimal": optimal,
-        "all_configs": results
-    }
-
-
-# Find process ID of the existing browser using os
-def find_process_by_port(port: int) -> str:
-    """Find process ID listening on a specific port.
-    
-    Args:
-        port: Port number to check
-        
-    Returns:
-        str: Process ID or empty string if not found
-    """
-    try:
-        if is_windows():
-            cmd = f"netstat -ano | findstr :{port}"
-            result = subprocess.check_output(cmd, shell=True).decode()
-            return result.strip().split()[-1] if result else ""
-        else:
-            cmd = f"lsof -i :{port} -t"
-            return subprocess.check_output(cmd, shell=True).decode().strip()
-    except subprocess.CalledProcessError:
-        return ""
-    
-async def check_process_is_running(process: subprocess.Popen, delay: float = 0.5) -> Tuple[bool, Optional[int], bytes, bytes]:
-    """Perform a quick check to make sure the browser started successfully."""
-    if not process:
-        return False, None, b"", b""
-        
-    # Check that process started without immediate termination
-    await asyncio.sleep(delay)
-    if process.poll() is not None:
-        # Process already terminated
-        stdout, stderr = b"", b""
-        try:
-            stdout, stderr = process.communicate(timeout=0.5)
-        except subprocess.TimeoutExpired:
-            pass
-
-        return False, process.returncode, stdout, stderr
-            
-
-    return True, 0, b"", b""
-
-
-def terminate_process(
-    pid: Union[int, str], 
-    timeout: float = 5.0,
-    force_kill_timeout: float = 3.0,
-    logger = None
-) -> Tuple[bool, Optional[str]]:
-    """
-    Robustly terminate a process across platforms with verification.
-    
-    Args:
-        pid: Process ID to terminate (int or string)
-        timeout: Seconds to wait for graceful termination before force killing
-        force_kill_timeout: Seconds to wait after force kill before considering it failed
-        logger: Optional logger object with error, warning, and info methods
-        
-    Returns:
-        Tuple of (success: bool, error_message: Optional[str])
-    """
-    # Convert pid to int if it's a string
-    if isinstance(pid, str):
-        try:
-            pid = int(pid)
-        except ValueError:
-            error_msg = f"Invalid PID format: {pid}"
-            if logger:
-                logger.error(error_msg)
-            return False, error_msg
-    
-    # Check if process exists
-    if not psutil.pid_exists(pid):
-        return True, None  # Process already terminated
-    
-    try:
-        process = psutil.Process(pid)
-        
-        # First attempt: graceful termination
-        if logger:
-            logger.info(f"Attempting graceful termination of process {pid}")
-            
-        if os.name == 'nt':  # Windows
-            subprocess.run(["taskkill", "/PID", str(pid)], 
-                          stdout=subprocess.DEVNULL, 
-                          stderr=subprocess.DEVNULL, 
-                          check=False)
-        else:  # Unix/Linux/MacOS
-            process.send_signal(signal.SIGTERM)
-        
-        # Wait for process to terminate
-        try:
-            process.wait(timeout=timeout)
-            if logger:
-                logger.info(f"Process {pid} terminated gracefully")
-            return True, None
-        except psutil.TimeoutExpired:
-            if logger:
-                logger.warning(f"Process {pid} did not terminate gracefully within {timeout} seconds, forcing termination")
-        
-        # Second attempt: force kill
-        if os.name == 'nt':  # Windows
-            subprocess.run(["taskkill", "/F", "/PID", str(pid)], 
-                          stdout=subprocess.DEVNULL, 
-                          stderr=subprocess.DEVNULL, 
-                          check=False)
-        else:  # Unix/Linux/MacOS
-            process.send_signal(signal.SIGKILL)
-        
-        # Verify process is killed
-        gone, alive = psutil.wait_procs([process], timeout=force_kill_timeout)
-        if process in alive:
-            error_msg = f"Failed to kill process {pid} even after force kill"
-            if logger:
-                logger.error(error_msg)
-            return False, error_msg
-            
-        if logger:
-            logger.info(f"Process {pid} terminated by force")
-        return True, None
-        
-    except psutil.NoSuchProcess:
-        # Process terminated while we were working with it
-        if logger:
-            logger.info(f"Process {pid} already terminated")
-        return True, None
-        
-    except Exception as e:
-        error_msg = f"Error terminating process {pid}: {str(e)}"
-        if logger:
-            logger.error(error_msg)
-        return False, error_msg

crawl4ai/browser_adapter.py

@@ -0,0 +1,293 @@
+# browser_adapter.py
+"""
+Browser adapter for Crawl4AI to support both Playwright and undetected browsers
+with minimal changes to existing codebase.
+"""
+
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any, Optional, Callable
+import time
+import json
+
+# Import both, but use conditionally
+try:
+    from playwright.async_api import Page
+except ImportError:
+    Page = Any
+
+try:
+    from patchright.async_api import Page as UndetectedPage
+except ImportError:
+    UndetectedPage = Any
+
+
+class BrowserAdapter(ABC):
+    """Abstract adapter for browser-specific operations"""
+    
+    @abstractmethod
+    async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any:
+        """Execute JavaScript in the page"""
+        pass
+    
+    @abstractmethod
+    async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
+        """Setup console message capturing, returns handler function if needed"""
+        pass
+    
+    @abstractmethod
+    async def setup_error_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
+        """Setup error capturing, returns handler function if needed"""
+        pass
+    
+    @abstractmethod
+    async def retrieve_console_messages(self, page: Page) -> List[Dict]:
+        """Retrieve captured console messages (for undetected browsers)"""
+        pass
+    
+    @abstractmethod
+    async def cleanup_console_capture(self, page: Page, handle_console: Optional[Callable], handle_error: Optional[Callable]):
+        """Clean up console event listeners"""
+        pass
+    
+    @abstractmethod
+    def get_imports(self) -> tuple:
+        """Get the appropriate imports for this adapter"""
+        pass
+
+
+class PlaywrightAdapter(BrowserAdapter):
+    """Adapter for standard Playwright"""
+    
+    async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any:
+        """Standard Playwright evaluate"""
+        if arg is not None:
+            return await page.evaluate(expression, arg)
+        return await page.evaluate(expression)
+    
+    async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
+        """Setup console capture using Playwright's event system"""
+        def handle_console_capture(msg):
+            try:
+                message_type = "unknown"
+                try:
+                    message_type = msg.type
+                except:
+                    pass
+                    
+                message_text = "unknown"
+                try:
+                    message_text = msg.text
+                except:
+                    pass
+                    
+                entry = {
+                    "type": message_type,
+                    "text": message_text,
+                    "timestamp": time.time()
+                }
+                
+                captured_console.append(entry)
+                
+            except Exception as e:
+                captured_console.append({
+                    "type": "console_capture_error", 
+                    "error": str(e), 
+                    "timestamp": time.time()
+                })
+        
+        page.on("console", handle_console_capture)
+        return handle_console_capture
+    
+    async def setup_error_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
+        """Setup error capture using Playwright's event system"""
+        def handle_pageerror_capture(err):
+            try:
+                error_message = "Unknown error"
+                try:
+                    error_message = err.message
+                except:
+                    pass
+                    
+                error_stack = ""
+                try:
+                    error_stack = err.stack
+                except:
+                    pass
+                    
+                captured_console.append({
+                    "type": "error",
+                    "text": error_message,
+                    "stack": error_stack,
+                    "timestamp": time.time()
+                })
+            except Exception as e:
+                captured_console.append({
+                    "type": "pageerror_capture_error", 
+                    "error": str(e), 
+                    "timestamp": time.time()
+                })
+        
+        page.on("pageerror", handle_pageerror_capture)
+        return handle_pageerror_capture
+    
+    async def retrieve_console_messages(self, page: Page) -> List[Dict]:
+        """Not needed for Playwright - messages are captured via events"""
+        return []
+    
+    async def cleanup_console_capture(self, page: Page, handle_console: Optional[Callable], handle_error: Optional[Callable]):
+        """Remove event listeners"""
+        if handle_console:
+            page.remove_listener("console", handle_console)
+        if handle_error:
+            page.remove_listener("pageerror", handle_error)
+    
+    def get_imports(self) -> tuple:
+        """Return Playwright imports"""
+        from playwright.async_api import Page, Error
+        from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+        return Page, Error, PlaywrightTimeoutError
+
+
+class UndetectedAdapter(BrowserAdapter):
+    """Adapter for undetected browser automation with stealth features"""
+    
+    def __init__(self):
+        self._console_script_injected = {}
+    
+    async def evaluate(self, page: UndetectedPage, expression: str, arg: Any = None) -> Any:
+        """Undetected browser evaluate with isolated context"""
+        # For most evaluations, use isolated context for stealth
+        # Only use non-isolated when we need to access our injected console capture
+        isolated = not (
+            "__console" in expression or 
+            "__captured" in expression or
+            "__error" in expression or
+            "window.__" in expression
+        )
+        
+        if arg is not None:
+            return await page.evaluate(expression, arg, isolated_context=isolated)
+        return await page.evaluate(expression, isolated_context=isolated)
+    
+    async def setup_console_capture(self, page: UndetectedPage, captured_console: List[Dict]) -> Optional[Callable]:
+        """Setup console capture using JavaScript injection for undetected browsers"""
+        if not self._console_script_injected.get(page, False):
+            await page.add_init_script("""
+                // Initialize console capture
+                window.__capturedConsole = [];
+                window.__capturedErrors = [];
+                
+                // Store original console methods
+                const originalConsole = {};
+                ['log', 'info', 'warn', 'error', 'debug'].forEach(method => {
+                    originalConsole[method] = console[method];
+                    console[method] = function(...args) {
+                        try {
+                            window.__capturedConsole.push({
+                                type: method,
+                                text: args.map(arg => {
+                                    try {
+                                        if (typeof arg === 'object') {
+                                            return JSON.stringify(arg);
+                                        }
+                                        return String(arg);
+                                    } catch (e) {
+                                        return '[Object]';
+                                    }
+                                }).join(' '),
+                                timestamp: Date.now()
+                            });
+                        } catch (e) {
+                            // Fail silently to avoid detection
+                        }
+                        
+                        // Call original method
+                        originalConsole[method].apply(console, args);
+                    };
+                });
+            """)
+            self._console_script_injected[page] = True
+        
+        return None  # No handler function needed for undetected browser
+    
+    async def setup_error_capture(self, page: UndetectedPage, captured_console: List[Dict]) -> Optional[Callable]:
+        """Setup error capture using JavaScript injection for undetected browsers"""
+        if not self._console_script_injected.get(page, False):
+            await page.add_init_script("""
+                // Capture errors
+                window.addEventListener('error', (event) => {
+                    try {
+                        window.__capturedErrors.push({
+                            type: 'error',
+                            text: event.message,
+                            stack: event.error ? event.error.stack : '',
+                            filename: event.filename,
+                            lineno: event.lineno,
+                            colno: event.colno,
+                            timestamp: Date.now()
+                        });
+                    } catch (e) {
+                        // Fail silently
+                    }
+                });
+                
+                // Capture unhandled promise rejections
+                window.addEventListener('unhandledrejection', (event) => {
+                    try {
+                        window.__capturedErrors.push({
+                            type: 'unhandledrejection',
+                            text: event.reason ? String(event.reason) : 'Unhandled Promise Rejection',
+                            stack: event.reason && event.reason.stack ? event.reason.stack : '',
+                            timestamp: Date.now()
+                        });
+                    } catch (e) {
+                        // Fail silently
+                    }
+                });
+            """)
+            self._console_script_injected[page] = True
+        
+        return None  # No handler function needed for undetected browser
+    
+    async def retrieve_console_messages(self, page: UndetectedPage) -> List[Dict]:
+        """Retrieve captured console messages and errors from the page"""
+        messages = []
+        
+        try:
+            # Get console messages
+            console_messages = await page.evaluate(
+                "() => { const msgs = window.__capturedConsole || []; window.__capturedConsole = []; return msgs; }",
+                isolated_context=False
+            )
+            messages.extend(console_messages)
+            
+            # Get errors
+            errors = await page.evaluate(
+                "() => { const errs = window.__capturedErrors || []; window.__capturedErrors = []; return errs; }",
+                isolated_context=False
+            )
+            messages.extend(errors)
+            
+            # Convert timestamps from JS to Python format
+            for msg in messages:
+                if 'timestamp' in msg and isinstance(msg['timestamp'], (int, float)):
+                    msg['timestamp'] = msg['timestamp'] / 1000.0  # Convert from ms to seconds
+                    
+        except Exception:
+            # If retrieval fails, return empty list
+            pass
+        
+        return messages
+    
+    async def cleanup_console_capture(self, page: UndetectedPage, handle_console: Optional[Callable], handle_error: Optional[Callable]):
+        """Clean up for undetected browser - retrieve final messages"""
+        # For undetected browser, we don't have event listeners to remove
+        # but we should retrieve any final messages
+        final_messages = await self.retrieve_console_messages(page)
+        return final_messages
+    
+    def get_imports(self) -> tuple:
+        """Return undetected browser imports"""
+        from patchright.async_api import Page, Error
+        from patchright.async_api import TimeoutError as PlaywrightTimeoutError
+        return Page, Error, PlaywrightTimeoutError

crawl4ai/browser_manager.py

@@ -5,29 +5,17 @@ import os
 import sys
 import shutil
 import tempfile
+import psutil  
+import signal
 import subprocess
+import shlex
 from playwright.async_api import BrowserContext
 import hashlib
 from .js_snippet import load_js_script
 from .config import DOWNLOAD_PAGE_TIMEOUT
 from .async_configs import BrowserConfig, CrawlerRunConfig
-from playwright_stealth import StealthConfig
 from .utils import get_chromium_path
 
-stealth_config = StealthConfig(
-    webdriver=True,
-    chrome_app=True,
-    chrome_csi=True,
-    chrome_load_times=True,
-    chrome_runtime=True,
-    navigator_languages=True,
-    navigator_plugins=True,
-    navigator_permissions=True,
-    webgl_vendor=True,
-    outerdimensions=True,
-    navigator_hardware_concurrency=True,
-    media_codecs=True,
-)
 
 BROWSER_DISABLE_OPTIONS = [
     "--disable-background-networking",
@@ -76,6 +64,51 @@ class ManagedBrowser:
             _cleanup(): Terminates the browser process and removes the temporary directory.
             create_profile(): Static method to create a user profile by launching a browser for user interaction.
     """
+    
+    @staticmethod
+    def build_browser_flags(config: BrowserConfig) -> List[str]:
+        """Common CLI flags for launching Chromium"""
+        flags = [
+            "--disable-gpu",
+            "--disable-gpu-compositing",
+            "--disable-software-rasterizer",
+            "--no-sandbox",
+            "--disable-dev-shm-usage",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--disable-infobars",
+            "--window-position=0,0",
+            "--ignore-certificate-errors",
+            "--ignore-certificate-errors-spki-list",
+            "--disable-blink-features=AutomationControlled",
+            "--window-position=400,0",
+            "--disable-renderer-backgrounding",
+            "--disable-ipc-flooding-protection",
+            "--force-color-profile=srgb",
+            "--mute-audio",
+            "--disable-background-timer-throttling",
+        ]
+        if config.light_mode:
+            flags.extend(BROWSER_DISABLE_OPTIONS)
+        if config.text_mode:
+            flags.extend([
+                "--blink-settings=imagesEnabled=false",
+                "--disable-remote-fonts",
+                "--disable-images",
+                "--disable-javascript",
+                "--disable-software-rasterizer",
+                "--disable-dev-shm-usage",
+            ])
+        # proxy support
+        if config.proxy:
+            flags.append(f"--proxy-server={config.proxy}")
+        elif config.proxy_config:
+            creds = ""
+            if config.proxy_config.username and config.proxy_config.password:
+                creds = f"{config.proxy_config.username}:{config.proxy_config.password}@"
+            flags.append(f"--proxy-server={creds}{config.proxy_config.server}")
+        # dedupe
+        return list(dict.fromkeys(flags))
 
     browser_type: str
     user_data_dir: str
@@ -94,6 +127,7 @@ class ManagedBrowser:
         host: str = "localhost",
         debugging_port: int = 9222,
         cdp_url: Optional[str] = None, 
+        browser_config: Optional[BrowserConfig] = None,
     ):
         """
         Initialize the ManagedBrowser instance.
@@ -109,17 +143,19 @@ class ManagedBrowser:
             host (str): Host for debugging the browser. Default: "localhost".
             debugging_port (int): Port for debugging the browser. Default: 9222.
             cdp_url (str or None): CDP URL to connect to the browser. Default: None.
+            browser_config (BrowserConfig): Configuration object containing all browser settings. Default: None.
         """
-        self.browser_type = browser_type
-        self.user_data_dir = user_data_dir
-        self.headless = headless
+        self.browser_type = browser_config.browser_type
+        self.user_data_dir = browser_config.user_data_dir
+        self.headless = browser_config.headless
         self.browser_process = None
         self.temp_dir = None
-        self.debugging_port = debugging_port
-        self.host = host
+        self.debugging_port = browser_config.debugging_port
+        self.host = browser_config.host
         self.logger = logger
         self.shutting_down = False
-        self.cdp_url = cdp_url
+        self.cdp_url = browser_config.cdp_url
+        self.browser_config = browser_config
 
     async def start(self) -> str:
         """
@@ -142,6 +178,48 @@ class ManagedBrowser:
         # Get browser path and args based on OS and browser type
         # browser_path = self._get_browser_path()
         args = await self._get_browser_args()
+        
+        if self.browser_config.extra_args:
+            args.extend(self.browser_config.extra_args)
+            
+
+        # ── make sure no old Chromium instance is owning the same port/profile ──
+        try:
+            if sys.platform == "win32":
+                if psutil is None:
+                    raise RuntimeError("psutil not available, cannot clean old browser")
+                for p in psutil.process_iter(["pid", "name", "cmdline"]):
+                    cl = " ".join(p.info.get("cmdline") or [])
+                    if (
+                        f"--remote-debugging-port={self.debugging_port}" in cl
+                        and f"--user-data-dir={self.user_data_dir}" in cl
+                    ):
+                        p.kill()
+                        p.wait(timeout=5)
+            else:  # macOS / Linux
+                # kill any process listening on the same debugging port
+                pids = (
+                    subprocess.check_output(shlex.split(f"lsof -t -i:{self.debugging_port}"))
+                    .decode()
+                    .strip()
+                    .splitlines()
+                )
+                for pid in pids:
+                    try:
+                        os.kill(int(pid), signal.SIGTERM)
+                    except ProcessLookupError:
+                        pass
+
+                # remove Chromium singleton locks, or new launch exits with
+                # “Opening in existing browser session.”
+                for f in ("SingletonLock", "SingletonSocket", "SingletonCookie"):
+                    fp = os.path.join(self.user_data_dir, f)
+                    if os.path.exists(fp):
+                        os.remove(fp)
+        except Exception as _e:
+            # non-fatal — we'll try to start anyway, but log what happened
+            self.logger.warning(f"pre-launch cleanup failed: {_e}", tag="BROWSER")            
+            
 
         # Start browser process
         try:
@@ -162,6 +240,13 @@ class ManagedBrowser:
                     preexec_fn=os.setpgrp  # Start in a new process group
                 )
                 
+            # If verbose is True print args used to run the process
+            if self.logger and self.browser_config.verbose:
+                self.logger.debug(
+                    f"Starting browser with args: {' '.join(args)}",
+                    tag="BROWSER"
+                )    
+                
             # We'll monitor for a short time to make sure it starts properly, but won't keep monitoring
             await asyncio.sleep(0.5)  # Give browser time to start
             await self._initial_startup_check()
@@ -274,29 +359,29 @@ class ManagedBrowser:
         return browser_path
 
     async def _get_browser_args(self) -> List[str]:
-        """Returns browser-specific command line arguments"""
-        base_args = [await self._get_browser_path()]
-
+        """Returns full CLI args for launching the browser"""
+        base = [await self._get_browser_path()]
         if self.browser_type == "chromium":
-            args = [
+            flags = [
                 f"--remote-debugging-port={self.debugging_port}",
                 f"--user-data-dir={self.user_data_dir}",
             ]
             if self.headless:
-                args.append("--headless=new")
+                flags.append("--headless=new")
+            # merge common launch flags
+            flags.extend(self.build_browser_flags(self.browser_config))
         elif self.browser_type == "firefox":
-            args = [
+            flags = [
                 "--remote-debugging-port",
                 str(self.debugging_port),
                 "--profile",
                 self.user_data_dir,
             ]
             if self.headless:
-                args.append("--headless")
+                flags.append("--headless")
         else:
             raise NotImplementedError(f"Browser type {self.browser_type} not supported")
-
-        return base_args + args
+        return base + flags
 
     async def cleanup(self):
         """Cleanup browser process and temporary directory"""
@@ -418,6 +503,56 @@ class ManagedBrowser:
         return profiler.delete_profile(profile_name_or_path)
 
 
+async def clone_runtime_state(
+    src: BrowserContext,
+    dst: BrowserContext,
+    crawlerRunConfig: CrawlerRunConfig | None = None,
+    browserConfig: BrowserConfig | None = None,
+) -> None:
+    """
+    Bring everything that *can* be changed at runtime from `src` → `dst`.
+
+    1. Cookies
+    2. localStorage (and sessionStorage, same API)
+    3. Extra headers, permissions, geolocation if supplied in configs
+    """
+
+    # ── 1. cookies ────────────────────────────────────────────────────────────
+    cookies = await src.cookies()
+    if cookies:
+        await dst.add_cookies(cookies)
+
+    # ── 2. localStorage / sessionStorage ──────────────────────────────────────
+    state = await src.storage_state()
+    for origin in state.get("origins", []):
+        url = origin["origin"]
+        kvs = origin.get("localStorage", [])
+        if not kvs:
+            continue
+
+        page = dst.pages[0] if dst.pages else await dst.new_page()
+        await page.goto(url, wait_until="domcontentloaded")
+        for k, v in kvs:
+            await page.evaluate("(k,v)=>localStorage.setItem(k,v)", k, v)
+
+    # ── 3. runtime-mutable extras from configs ────────────────────────────────
+    # headers
+    if browserConfig and browserConfig.headers:
+        await dst.set_extra_http_headers(browserConfig.headers)
+
+    # geolocation
+    if crawlerRunConfig and crawlerRunConfig.geolocation:
+        await dst.grant_permissions(["geolocation"])
+        await dst.set_geolocation(
+            {
+                "latitude": crawlerRunConfig.geolocation.latitude,
+                "longitude": crawlerRunConfig.geolocation.longitude,
+                "accuracy": crawlerRunConfig.geolocation.accuracy,
+            }
+        )
+        
+    return dst
+
 
 
 class BrowserManager:
@@ -438,22 +573,26 @@ class BrowserManager:
     _playwright_instance = None
     
     @classmethod
-    async def get_playwright(cls):
-        from playwright.async_api import async_playwright
-        if cls._playwright_instance is None:
-            cls._playwright_instance = await async_playwright().start()
+    async def get_playwright(cls, use_undetected: bool = False):
+        if use_undetected:
+            from patchright.async_api import async_playwright
+        else:
+            from playwright.async_api import async_playwright
+        cls._playwright_instance = await async_playwright().start()
         return cls._playwright_instance    
 
-    def __init__(self, browser_config: BrowserConfig, logger=None):
+    def __init__(self, browser_config: BrowserConfig, logger=None, use_undetected: bool = False):
         """
         Initialize the BrowserManager with a browser configuration.
 
         Args:
             browser_config (BrowserConfig): Configuration object containing all browser settings
             logger: Logger instance for recording events and errors
+            use_undetected (bool): Whether to use undetected browser (Patchright)
         """
         self.config: BrowserConfig = browser_config
         self.logger = logger
+        self.use_undetected = use_undetected
 
         # Browser state
         self.browser = None
@@ -467,7 +606,11 @@ class BrowserManager:
 
         # Keep track of contexts by a "config signature," so each unique config reuses a single context
         self.contexts_by_config = {}
-        self._contexts_lock = asyncio.Lock() 
+        self._contexts_lock = asyncio.Lock()
+        
+        # Stealth-related attributes
+        self._stealth_instance = None
+        self._stealth_cm = None 
 
         # Initialize ManagedBrowser if needed
         if self.config.use_managed_browser:
@@ -478,6 +621,7 @@ class BrowserManager:
                 logger=self.logger,
                 debugging_port=self.config.debugging_port,
                 cdp_url=self.config.cdp_url,
+                browser_config=self.config,
             )
 
     async def start(self):
@@ -492,10 +636,23 @@ class BrowserManager:
 
         Note: This method should be called in a separate task to avoid blocking the main event loop.
         """
-        self.playwright  = await self.get_playwright()
-        if self.playwright is None:
+        if self.playwright is not None:
+            await self.close()
+            
+        if self.use_undetected:
+            from patchright.async_api import async_playwright
+        else:
             from playwright.async_api import async_playwright
 
+        # Initialize playwright with or without stealth
+        if self.config.enable_stealth and not self.use_undetected:
+            # Import stealth only when needed
+            from playwright_stealth import Stealth
+            # Use the recommended stealth wrapper approach
+            self._stealth_instance = Stealth()
+            self._stealth_cm = self._stealth_instance.use_async(async_playwright())
+            self.playwright = await self._stealth_cm.__aenter__()
+        else:
             self.playwright = await async_playwright().start()
 
         if self.config.cdp_url or self.config.use_managed_browser:
@@ -565,6 +722,9 @@ class BrowserManager:
         if self.config.extra_args:
             args.extend(self.config.extra_args)
 
+        # Deduplicate args
+        args = list(dict.fromkeys(args))
+        
         browser_args = {"headless": self.config.headless, "args": args}
 
         if self.config.chrome_channel:
@@ -660,7 +820,7 @@ class BrowserManager:
                     "name": "cookiesEnabled",
                     "value": "true",
                     "url": crawlerRunConfig.url
-                    if crawlerRunConfig
+                    if crawlerRunConfig and crawlerRunConfig.url
                     else "https://crawl4ai.com/",
                 }
             ]
@@ -779,6 +939,23 @@ class BrowserManager:
             # Update context settings with text mode settings
             context_settings.update(text_mode_settings)
 
+        # inject locale / tz / geo if user provided them
+        if crawlerRunConfig:
+            if crawlerRunConfig.locale:
+                context_settings["locale"] = crawlerRunConfig.locale
+            if crawlerRunConfig.timezone_id:
+                context_settings["timezone_id"] = crawlerRunConfig.timezone_id
+            if crawlerRunConfig.geolocation:
+                context_settings["geolocation"] = {
+                    "latitude": crawlerRunConfig.geolocation.latitude,
+                    "longitude": crawlerRunConfig.geolocation.longitude,
+                    "accuracy": crawlerRunConfig.geolocation.accuracy,
+                }
+                # ensure geolocation permission
+                perms = context_settings.get("permissions", [])
+                perms.append("geolocation")
+                context_settings["permissions"] = perms
+
         # Create and return the context with all settings
         context = await self.browser.new_context(**context_settings)
 
@@ -811,6 +988,10 @@ class BrowserManager:
             "semaphore_count",
             "url"
         ]
+        
+        # Do NOT exclude locale, timezone_id, or geolocation as these DO affect browser context
+        # and should cause a new context to be created if they change
+        
         for key in ephemeral_keys:
             if key in config_dict:
                 del config_dict[key]
@@ -842,11 +1023,17 @@ class BrowserManager:
 
         # If using a managed browser, just grab the shared default_context
         if self.config.use_managed_browser:
-            context = self.default_context
-            pages = context.pages
-            page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
-            if not page:
-                page = await context.new_page()
+            if self.config.storage_state:
+                context = await self.create_browser_context(crawlerRunConfig)
+                ctx = self.default_context        # default context, one window only
+                ctx = await clone_runtime_state(context, ctx, crawlerRunConfig, self.config)
+                page = await ctx.new_page()
+            else:
+                context = self.default_context
+                pages = context.pages
+                page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+                if not page:
+                    page = context.pages[0] # await context.new_page()
         else:
             # Otherwise, check if we have an existing context for this config
             config_signature = self._make_config_signature(crawlerRunConfig)
@@ -928,5 +1115,19 @@ class BrowserManager:
             self.managed_browser = None
 
         if self.playwright:
-            await self.playwright.stop()
+            # Handle stealth context manager cleanup if it exists
+            if hasattr(self, '_stealth_cm') and self._stealth_cm is not None:
+                try:
+                    await self._stealth_cm.__aexit__(None, None, None)
+                except Exception as e:
+                    if self.logger:
+                        self.logger.error(
+                            message="Error closing stealth context: {error}",
+                            tag="ERROR", 
+                            params={"error": str(e)}
+                        )
+                self._stealth_cm = None
+                self._stealth_instance = None
+            else:
+                await self.playwright.stop()
             self.playwright = None

crawl4ai/browser_profiler.py

@@ -15,12 +15,12 @@ import shutil
 import json
 import subprocess
 import time
-from typing import List, Dict, Optional, Any, Tuple
-from colorama import Fore, Style, init
+from typing import List, Dict, Optional, Any
+from rich.console import Console
 
 from .async_configs import BrowserConfig
 from .browser_manager import ManagedBrowser
-from .async_logger import AsyncLogger, AsyncLoggerBase
+from .async_logger import AsyncLogger, AsyncLoggerBase, LogColor
 from .utils import get_home_folder
 
 
@@ -45,8 +45,8 @@ class BrowserProfiler:
             logger (AsyncLoggerBase, optional): Logger for outputting messages.
                 If None, a default AsyncLogger will be created.
         """
-        # Initialize colorama for colorful terminal output
-        init()
+        # Initialize rich console for colorful input prompts
+        self.console = Console()
         
         # Create a logger if not provided
         if logger is None:
@@ -127,26 +127,30 @@ class BrowserProfiler:
         profile_path = os.path.join(self.profiles_dir, profile_name)
         os.makedirs(profile_path, exist_ok=True)
         
-        # Print instructions for the user with colorama formatting
-        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
-        self.logger.info(f"\n{border}", tag="PROFILE")
-        self.logger.info(f"Creating browser profile: {Fore.GREEN}{profile_name}{Style.RESET_ALL}", tag="PROFILE")
-        self.logger.info(f"Profile directory: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+        # Print instructions for the user with rich formatting
+        border = f"{'='*80}"
+        self.logger.info("{border}", tag="PROFILE", params={"border": f"\n{border}"}, colors={"border": LogColor.CYAN})
+        self.logger.info("Creating browser profile: {profile_name}", tag="PROFILE", params={"profile_name": profile_name}, colors={"profile_name": LogColor.GREEN})
+        self.logger.info("Profile directory: {profile_path}", tag="PROFILE", params={"profile_path": profile_path}, colors={"profile_path": LogColor.YELLOW})
         
         self.logger.info("\nInstructions:", tag="PROFILE")
         self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE")
-        self.logger.info(f"2. {Fore.CYAN}Log in to websites{Style.RESET_ALL}, configure settings, etc. as needed.", tag="PROFILE")
-        self.logger.info(f"3. When you're done, {Fore.YELLOW}press 'q' in this terminal{Style.RESET_ALL} to close the browser.", tag="PROFILE")
+        self.logger.info("{segment}, configure settings, etc. as needed.", tag="PROFILE", params={"segment": "2. Log in to websites"}, colors={"segment": LogColor.CYAN})
+        self.logger.info("3. When you're done, {segment} to close the browser.", tag="PROFILE", params={"segment": "press 'q' in this terminal"}, colors={"segment": LogColor.YELLOW})
         self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE")
-        self.logger.info(f"{border}\n", tag="PROFILE")
+        self.logger.info("{border}", tag="PROFILE", params={"border": f"{border}\n"}, colors={"border": LogColor.CYAN})
+        
+        browser_config.headless = False
+        browser_config.user_data_dir = profile_path
+        
         
         # Create managed browser instance
         managed_browser = ManagedBrowser(
-            browser_type=browser_config.browser_type,
-            user_data_dir=profile_path,
-            headless=False,  # Must be visible
+            browser_config=browser_config,
+            # user_data_dir=profile_path,
+            # headless=False,  # Must be visible
             logger=self.logger,
-            debugging_port=browser_config.debugging_port
+            # debugging_port=browser_config.debugging_port
         )
         
         # Set up signal handlers to ensure cleanup on interrupt
@@ -181,7 +185,7 @@ class BrowserProfiler:
             import select
             
             # First output the prompt
-            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' when you've finished using the browser...{Style.RESET_ALL}", tag="PROFILE")
+            self.logger.info("Press 'q' when you've finished using the browser...", tag="PROFILE")
             
             # Save original terminal settings
             fd = sys.stdin.fileno()
@@ -197,7 +201,7 @@ class BrowserProfiler:
                     if readable:
                         key = sys.stdin.read(1)
                         if key.lower() == 'q':
-                            self.logger.info(f"{Fore.GREEN}Closing browser and saving profile...{Style.RESET_ALL}", tag="PROFILE")
+                            self.logger.info("Closing browser and saving profile...", tag="PROFILE", base_color=LogColor.GREEN)
                             user_done_event.set()
                             return
                     
@@ -214,8 +218,18 @@ class BrowserProfiler:
                 termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
         
         try:
+            from playwright.async_api import async_playwright
+
             # Start the browser
-            await managed_browser.start()
+            # await managed_browser.start()
+            # 1. ── Start the browser ─────────────────────────────────────────
+            cdp_url = await managed_browser.start()
+
+            # 2. ── Attach Playwright to that running Chrome ──────────────────
+            pw       = await async_playwright().start()
+            browser  = await pw.chromium.connect_over_cdp(cdp_url)
+            # Grab the existing default context (there is always one)
+            context  = browser.contexts[0]
             
             # Check if browser started successfully
             browser_process = managed_browser.browser_process
@@ -223,7 +237,7 @@ class BrowserProfiler:
                 self.logger.error("Failed to start browser process.", tag="PROFILE")
                 return None
             
-            self.logger.info(f"Browser launched. {Fore.CYAN}Waiting for you to finish...{Style.RESET_ALL}", tag="PROFILE") 
+            self.logger.info("Browser launched. Waiting for you to finish...", tag="PROFILE") 
             
             # Start listening for keyboard input
             listener_task = asyncio.create_task(listen_for_quit_command())
@@ -240,15 +254,27 @@ class BrowserProfiler:
                 except asyncio.CancelledError:
                     pass
             
+            # 3. ── Persist storage state *before* we kill Chrome ─────────────
+            state_file = os.path.join(profile_path, "storage_state.json")
+            try:
+                await context.storage_state(path=state_file)
+                self.logger.info(f"[PROFILE].i  storage_state saved → {state_file}", tag="PROFILE")
+            except Exception as e:
+                self.logger.warning(f"[PROFILE].w  failed to save storage_state: {e}", tag="PROFILE")
+
+            # 4. ── Close everything cleanly ──────────────────────────────────
+            await browser.close()
+            await pw.stop()
+
             # If the browser is still running and the user pressed 'q', terminate it
             if browser_process.poll() is None and user_done_event.is_set():
                 self.logger.info("Terminating browser process...", tag="PROFILE")
                 await managed_browser.cleanup()
             
-            self.logger.success(f"Browser closed. Profile saved at: {Fore.GREEN}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+            self.logger.success(f"Browser closed. Profile saved at: {profile_path}", tag="PROFILE")
                 
         except Exception as e:
-            self.logger.error(f"Error creating profile: {str(e)}", tag="PROFILE")
+            self.logger.error(f"Error creating profile: {e!s}", tag="PROFILE")
             await managed_browser.cleanup()
             return None
         finally:
@@ -440,25 +466,27 @@ class BrowserProfiler:
             ```
         """
         while True:
-            self.logger.info(f"\n{Fore.CYAN}Profile Management Options:{Style.RESET_ALL}", tag="MENU")
-            self.logger.info(f"1. {Fore.GREEN}Create a new profile{Style.RESET_ALL}", tag="MENU")
-            self.logger.info(f"2. {Fore.YELLOW}List available profiles{Style.RESET_ALL}", tag="MENU")
-            self.logger.info(f"3. {Fore.RED}Delete a profile{Style.RESET_ALL}", tag="MENU")
+            self.logger.info("\nProfile Management Options:", tag="MENU")
+            self.logger.info("1. Create a new profile", tag="MENU", base_color=LogColor.GREEN)
+            self.logger.info("2. List available profiles", tag="MENU", base_color=LogColor.YELLOW)
+            self.logger.info("3. Delete a profile", tag="MENU", base_color=LogColor.RED)
             
             # Only show crawl option if callback provided
             if crawl_callback:
-                self.logger.info(f"4. {Fore.CYAN}Use a profile to crawl a website{Style.RESET_ALL}", tag="MENU")
-                self.logger.info(f"5. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                self.logger.info("4. Use a profile to crawl a website", tag="MENU", base_color=LogColor.CYAN)
+                self.logger.info("5. Exit", tag="MENU", base_color=LogColor.MAGENTA)
                 exit_option = "5"
             else:
-                self.logger.info(f"4. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                self.logger.info("4. Exit", tag="MENU", base_color=LogColor.MAGENTA)
                 exit_option = "4"
             
-            choice = input(f"\n{Fore.CYAN}Enter your choice (1-{exit_option}): {Style.RESET_ALL}")
+            self.logger.info(f"\n[cyan]Enter your choice (1-{exit_option}): [/cyan]", end="")
+            choice = input()
             
             if choice == "1":
                 # Create new profile
-                name = input(f"{Fore.GREEN}Enter a name for the new profile (or press Enter for auto-generated name): {Style.RESET_ALL}")
+                self.console.print("[green]Enter a name for the new profile (or press Enter for auto-generated name): [/green]", end="")
+                name = input()
                 await self.create_profile(name or None)
                 
             elif choice == "2":
@@ -469,11 +497,11 @@ class BrowserProfiler:
                     self.logger.warning("  No profiles found. Create one first with option 1.", tag="PROFILES")
                     continue
                 
-                # Print profile information with colorama formatting
+                # Print profile information 
                 self.logger.info("\nAvailable profiles:", tag="PROFILES")
                 for i, profile in enumerate(profiles):
-                    self.logger.info(f"[{i+1}] {Fore.CYAN}{profile['name']}{Style.RESET_ALL}", tag="PROFILES")
-                    self.logger.info(f"    Path: {Fore.YELLOW}{profile['path']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    self.logger.info(f"    Path: {profile['path']}", tag="PROFILES", base_color=LogColor.YELLOW)
                     self.logger.info(f"    Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES")
                     self.logger.info(f"    Browser type: {profile['type']}", tag="PROFILES")
                     self.logger.info("", tag="PROFILES")  # Empty line for spacing
@@ -486,12 +514,13 @@ class BrowserProfiler:
                     continue
                     
                 # Display numbered list
-                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                self.logger.info("\nAvailable profiles:", tag="PROFILES", base_color=LogColor.YELLOW)
                 for i, profile in enumerate(profiles):
                     self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
                     
                 # Get profile to delete
-                profile_idx = input(f"{Fore.RED}Enter the number of the profile to delete (or 'c' to cancel): {Style.RESET_ALL}")
+                self.console.print("[red]Enter the number of the profile to delete (or 'c' to cancel): [/red]", end="")
+                profile_idx = input()
                 if profile_idx.lower() == 'c':
                     continue
                     
@@ -499,17 +528,18 @@ class BrowserProfiler:
                     idx = int(profile_idx) - 1
                     if 0 <= idx < len(profiles):
                         profile_name = profiles[idx]["name"]
-                        self.logger.info(f"Deleting profile: {Fore.YELLOW}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                        self.logger.info(f"Deleting profile: [yellow]{profile_name}[/yellow]", tag="PROFILES")
                         
                         # Confirm deletion
-                        confirm = input(f"{Fore.RED}Are you sure you want to delete this profile? (y/n): {Style.RESET_ALL}")
+                        self.console.print("[red]Are you sure you want to delete this profile? (y/n): [/red]", end="")
+                        confirm = input()
                         if confirm.lower() == 'y':
                             success = self.delete_profile(profiles[idx]["path"])
                             
                             if success:
-                                self.logger.success(f"Profile {Fore.GREEN}{profile_name}{Style.RESET_ALL} deleted successfully", tag="PROFILES")
+                                self.logger.success(f"Profile {profile_name} deleted successfully", tag="PROFILES")
                             else:
-                                self.logger.error(f"Failed to delete profile {Fore.RED}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                                self.logger.error(f"Failed to delete profile {profile_name}", tag="PROFILES")
                     else:
                         self.logger.error("Invalid profile number", tag="PROFILES")
                 except ValueError:
@@ -523,12 +553,13 @@ class BrowserProfiler:
                     continue
                     
                 # Display numbered list
-                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                self.logger.info("\nAvailable profiles:", tag="PROFILES", base_color=LogColor.YELLOW)
                 for i, profile in enumerate(profiles):
                     self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
                     
                 # Get profile to use
-                profile_idx = input(f"{Fore.CYAN}Enter the number of the profile to use (or 'c' to cancel): {Style.RESET_ALL}")
+                self.console.print("[cyan]Enter the number of the profile to use (or 'c' to cancel): [/cyan]", end="")
+                profile_idx = input()
                 if profile_idx.lower() == 'c':
                     continue
                     
@@ -536,7 +567,8 @@ class BrowserProfiler:
                     idx = int(profile_idx) - 1
                     if 0 <= idx < len(profiles):
                         profile_path = profiles[idx]["path"]
-                        url = input(f"{Fore.CYAN}Enter the URL to crawl: {Style.RESET_ALL}")
+                        self.console.print("[cyan]Enter the URL to crawl: [/cyan]", end="")
+                        url = input()
                         if url:
                             # Call the provided crawl callback
                             await crawl_callback(profile_path, url)
@@ -597,17 +629,26 @@ class BrowserProfiler:
             os.makedirs(profile_path, exist_ok=True)
         
         # Print initial information
-        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
-        self.logger.info(f"\n{border}", tag="CDP")
-        self.logger.info(f"Launching standalone browser with CDP debugging", tag="CDP")
-        self.logger.info(f"Browser type: {Fore.GREEN}{browser_type}{Style.RESET_ALL}", tag="CDP")
-        self.logger.info(f"Profile path: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="CDP")
-        self.logger.info(f"Debugging port: {Fore.CYAN}{debugging_port}{Style.RESET_ALL}", tag="CDP")
-        self.logger.info(f"Headless mode: {Fore.CYAN}{headless}{Style.RESET_ALL}", tag="CDP")
+        border = f"{'='*80}"
+        self.logger.info("{border}", tag="CDP", params={"border": border}, colors={"border": LogColor.CYAN})
+        self.logger.info("Launching standalone browser with CDP debugging", tag="CDP")
+        self.logger.info("Browser type: {browser_type}", tag="CDP", params={"browser_type": browser_type}, colors={"browser_type": LogColor.CYAN})
+        self.logger.info("Profile path: {profile_path}", tag="CDP", params={"profile_path": profile_path}, colors={"profile_path": LogColor.YELLOW})
+        self.logger.info(f"Debugging port: {debugging_port}", tag="CDP")
+        self.logger.info(f"Headless mode: {headless}", tag="CDP")
+        
+        # create browser config
+        browser_config = BrowserConfig(
+            browser_type=browser_type,
+            headless=headless,
+            user_data_dir=profile_path,
+            debugging_port=debugging_port,
+            verbose=True
+        )
         
         # Create managed browser instance
         managed_browser = ManagedBrowser(
-            browser_type=browser_type,
+            browser_config=browser_config,
             user_data_dir=profile_path,
             headless=headless,
             logger=self.logger,
@@ -646,7 +687,7 @@ class BrowserProfiler:
             import select
             
             # First output the prompt
-            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' to stop the browser and exit...{Style.RESET_ALL}", tag="CDP")
+            self.logger.info("Press 'q' to stop the browser and exit...", tag="CDP")
             
             # Save original terminal settings
             fd = sys.stdin.fileno()
@@ -662,7 +703,7 @@ class BrowserProfiler:
                     if readable:
                         key = sys.stdin.read(1)
                         if key.lower() == 'q':
-                            self.logger.info(f"{Fore.GREEN}Closing browser...{Style.RESET_ALL}", tag="CDP")
+                            self.logger.info("Closing browser...", tag="CDP")
                             user_done_event.set()
                             return
                     
@@ -716,20 +757,20 @@ class BrowserProfiler:
                 self.logger.error("Failed to start browser process.", tag="CDP")
                 return None
             
-            self.logger.info(f"Browser launched successfully. Retrieving CDP information...", tag="CDP") 
+            self.logger.info("Browser launched successfully. Retrieving CDP information...", tag="CDP") 
             
             # Get CDP URL and JSON config
             cdp_url, config_json = await get_cdp_json(debugging_port)
             
             if cdp_url:
-                self.logger.success(f"CDP URL: {Fore.GREEN}{cdp_url}{Style.RESET_ALL}", tag="CDP")
+                self.logger.success(f"CDP URL: {cdp_url}", tag="CDP")
                 
                 if config_json:
                     # Display relevant CDP information
-                    self.logger.info(f"Browser: {Fore.CYAN}{config_json.get('Browser', 'Unknown')}{Style.RESET_ALL}", tag="CDP")
-                    self.logger.info(f"Protocol Version: {config_json.get('Protocol-Version', 'Unknown')}", tag="CDP")
+                    self.logger.info(f"Browser: {config_json.get('Browser', 'Unknown')}", tag="CDP", colors={"Browser": LogColor.CYAN})
+                    self.logger.info(f"Protocol Version: {config_json.get('Protocol-Version', 'Unknown')}", tag="CDP", colors={"Protocol-Version": LogColor.CYAN})
                     if 'webSocketDebuggerUrl' in config_json:
-                        self.logger.info(f"WebSocket URL: {Fore.GREEN}{config_json['webSocketDebuggerUrl']}{Style.RESET_ALL}", tag="CDP")
+                        self.logger.info("WebSocket URL: {webSocketDebuggerUrl}", tag="CDP", params={"webSocketDebuggerUrl": config_json['webSocketDebuggerUrl']}, colors={"webSocketDebuggerUrl": LogColor.GREEN})
                 else:
                     self.logger.warning("Could not retrieve CDP configuration JSON", tag="CDP")
             else:
@@ -757,7 +798,7 @@ class BrowserProfiler:
                 self.logger.info("Terminating browser process...", tag="CDP")
                 await managed_browser.cleanup()
             
-            self.logger.success(f"Browser closed.", tag="CDP")
+            self.logger.success("Browser closed.", tag="CDP")
                 
         except Exception as e:
             self.logger.error(f"Error launching standalone browser: {str(e)}", tag="CDP")
@@ -972,3 +1013,30 @@ class BrowserProfiler:
             'info': browser_info
         }
 
+
+if __name__ == "__main__":
+    # Example usage
+    profiler = BrowserProfiler()
+    
+    # Create a new profile
+    import os
+    from pathlib import Path
+    home_dir = Path.home()
+    profile_path = asyncio.run(profiler.create_profile( str(home_dir / ".crawl4ai/profiles/test-profile")))
+
+        
+            
+    # Launch a standalone browser
+    asyncio.run(profiler.launch_standalone_browser())
+    
+    # List profiles
+    profiles = profiler.list_profiles()
+    for profile in profiles:
+        print(f"Profile: {profile['name']}, Path: {profile['path']}")
+    
+    # Delete a profile
+    success = profiler.delete_profile("my-profile")
+    if success:
+        print("Profile deleted successfully")
+    else:
+        print("Failed to delete profile")

crawl4ai/cli.py

@@ -27,7 +27,10 @@ from crawl4ai import (
     PruningContentFilter,
     BrowserProfiler,
     DefaultMarkdownGenerator,
-    LLMConfig
+    LLMConfig,
+    BFSDeepCrawlStrategy,
+    DFSDeepCrawlStrategy,
+    BestFirstCrawlingStrategy,
 )
 from crawl4ai.config import USER_SETTINGS
 from litellm import completion
@@ -1010,13 +1013,15 @@ def cdp_cmd(user_data_dir: Optional[str], port: int, browser_type: str, headless
 @click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2")
 @click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all")
 @click.option("--output-file", "-O", type=click.Path(), help="Output file path (default: stdout)")
-@click.option("--bypass-cache", "-b", is_flag=True, default=True, help="Bypass cache when crawling")
+@click.option("--bypass-cache", "-bc", is_flag=True, default=True, help="Bypass cache when crawling")
 @click.option("--question", "-q", help="Ask a question about the crawled content")
 @click.option("--verbose", "-v", is_flag=True)
 @click.option("--profile", "-p", help="Use a specific browser profile (by name)")
+@click.option("--deep-crawl", type=click.Choice(["bfs", "dfs", "best-first"]), help="Enable deep crawling with specified strategy (bfs, dfs, or best-first)")
+@click.option("--max-pages", type=int, default=10, help="Maximum number of pages to crawl in deep crawl mode")
 def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config: str, 
            extraction_config: str, json_extract: str, schema: str, browser: Dict, crawler: Dict,
-           output: str, output_file: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
+           output: str, output_file: str, bypass_cache: bool, question: str, verbose: bool, profile: str, deep_crawl: str, max_pages: int):
     """Crawl a website and extract content
     
     Simple Usage:
@@ -1073,7 +1078,8 @@ def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config:
                 crawler_cfg.markdown_generator = DefaultMarkdownGenerator(
                     content_filter = BM25ContentFilter(
                         user_query=filter_conf.get("query"),
-                        bm25_threshold=filter_conf.get("threshold", 1.0)
+                        bm25_threshold=filter_conf.get("threshold", 1.0),
+                        use_stemming=filter_conf.get("use_stemming", True),
                     )
                 )
             elif filter_conf["type"] == "pruning":
@@ -1155,6 +1161,27 @@ Always return valid, properly formatted JSON."""
 
         crawler_cfg.scraping_strategy = LXMLWebScrapingStrategy()    
 
+        # Handle deep crawling configuration
+        if deep_crawl:
+            if deep_crawl == "bfs":
+                crawler_cfg.deep_crawl_strategy = BFSDeepCrawlStrategy(
+                    max_depth=3,
+                    max_pages=max_pages
+                )
+            elif deep_crawl == "dfs":
+                crawler_cfg.deep_crawl_strategy = DFSDeepCrawlStrategy(
+                    max_depth=3,
+                    max_pages=max_pages
+                )
+            elif deep_crawl == "best-first":
+                crawler_cfg.deep_crawl_strategy = BestFirstCrawlingStrategy(
+                    max_depth=3,
+                    max_pages=max_pages
+                )
+            
+            if verbose:
+                console.print(f"[green]Deep crawling enabled:[/green] {deep_crawl} strategy, max {max_pages} pages")
+
         config = get_global_config()
         
         browser_cfg.verbose = config.get("VERBOSE", False)
@@ -1169,39 +1196,60 @@ Always return valid, properly formatted JSON."""
             verbose
         )
 
+        # Handle deep crawl results (list) vs single result
+        if isinstance(result, list):
+            if len(result) == 0:
+                click.echo("No results found during deep crawling")
+                return
+            # Use the first result for question answering and output
+            main_result = result[0]
+            all_results = result
+        else:
+            # Single result from regular crawling
+            main_result = result
+            all_results = [result]
+
         # Handle question
         if question:
             provider, token = setup_llm_config()
-            markdown = result.markdown.raw_markdown
+            markdown = main_result.markdown.raw_markdown
             anyio.run(stream_llm_response, url, markdown, question, provider, token)
             return
         
         # Handle output
         if not output_file:
             if output == "all":
-                click.echo(json.dumps(result.model_dump(), indent=2))
+                if isinstance(result, list):
+                    output_data = [r.model_dump() for r in all_results]
+                    click.echo(json.dumps(output_data, indent=2))
+                else:
+                    click.echo(json.dumps(main_result.model_dump(), indent=2))
             elif output == "json":
-                print(result.extracted_content)
-                extracted_items = json.loads(result.extracted_content)
+                print(main_result.extracted_content)
+                extracted_items = json.loads(main_result.extracted_content)
                 click.echo(json.dumps(extracted_items, indent=2))
                 
             elif output in ["markdown", "md"]:
-                click.echo(result.markdown.raw_markdown)
+                click.echo(main_result.markdown.raw_markdown)
             elif output in ["markdown-fit", "md-fit"]:
-                click.echo(result.markdown.fit_markdown)
+                click.echo(main_result.markdown.fit_markdown)
         else:
             if output == "all":
                 with open(output_file, "w") as f:
-                    f.write(json.dumps(result.model_dump(), indent=2))
+                    if isinstance(result, list):
+                        output_data = [r.model_dump() for r in all_results]
+                        f.write(json.dumps(output_data, indent=2))
+                    else:
+                        f.write(json.dumps(main_result.model_dump(), indent=2))
             elif output == "json":
                 with open(output_file, "w") as f:
-                    f.write(result.extracted_content)
+                    f.write(main_result.extracted_content)
             elif output in ["markdown", "md"]:
                 with open(output_file, "w") as f:
-                    f.write(result.markdown.raw_markdown)
+                    f.write(main_result.markdown.raw_markdown)
             elif output in ["markdown-fit", "md-fit"]:
                 with open(output_file, "w") as f:
-                    f.write(result.markdown.fit_markdown)
+                    f.write(main_result.markdown.fit_markdown)
             
     except Exception as e:
         raise click.ClickException(str(e))
@@ -1353,9 +1401,11 @@ def profiles_cmd():
 @click.option("--question", "-q", help="Ask a question about the crawled content")
 @click.option("--verbose", "-v", is_flag=True)
 @click.option("--profile", "-p", help="Use a specific browser profile (by name)")
+@click.option("--deep-crawl", type=click.Choice(["bfs", "dfs", "best-first"]), help="Enable deep crawling with specified strategy")
+@click.option("--max-pages", type=int, default=10, help="Maximum number of pages to crawl in deep crawl mode")
 def default(url: str, example: bool, browser_config: str, crawler_config: str, filter_config: str, 
         extraction_config: str, json_extract: str, schema: str, browser: Dict, crawler: Dict,
-        output: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
+        output: str, bypass_cache: bool, question: str, verbose: bool, profile: str, deep_crawl: str, max_pages: int):
     """Crawl4AI CLI - Web content extraction tool
 
     Simple Usage:
@@ -1405,7 +1455,9 @@ def default(url: str, example: bool, browser_config: str, crawler_config: str, f
         bypass_cache=bypass_cache,
         question=question,
         verbose=verbose,
-        profile=profile
+        profile=profile,
+        deep_crawl=deep_crawl,
+        max_pages=max_pages
     )
 
 def main():

crawl4ai/config.py

@@ -29,6 +29,14 @@ PROVIDER_MODELS = {
     'gemini/gemini-2.0-flash-lite-preview-02-05': os.getenv("GEMINI_API_KEY"),
     "deepseek/deepseek-chat": os.getenv("DEEPSEEK_API_KEY"),
 }
+PROVIDER_MODELS_PREFIXES = {
+    "ollama": "no-token-needed",  # Any model from Ollama no need for API token
+    "groq": os.getenv("GROQ_API_KEY"),
+    "openai": os.getenv("OPENAI_API_KEY"),
+    "anthropic": os.getenv("ANTHROPIC_API_KEY"),
+    "gemini": os.getenv("GEMINI_API_KEY"),
+    "deepseek": os.getenv("DEEPSEEK_API_KEY"),
+}
 
 # Chunk token threshold
 CHUNK_TOKEN_THRESHOLD = 2**11  # 2048 tokens

crawl4ai/content_filter_strategy.py

@@ -27,8 +27,7 @@ import json
 import hashlib
 from pathlib import Path
 from concurrent.futures import ThreadPoolExecutor
-from .async_logger import AsyncLogger, LogLevel
-from colorama import Fore, Style
+from .async_logger import AsyncLogger, LogLevel, LogColor
 
 
 class RelevantContentFilter(ABC):
@@ -406,6 +405,7 @@ class BM25ContentFilter(RelevantContentFilter):
         user_query: str = None,
         bm25_threshold: float = 1.0,
         language: str = "english",
+        use_stemming: bool = True,
     ):
         """
         Initializes the BM25ContentFilter class, if not provided, falls back to page metadata.
@@ -417,9 +417,11 @@ class BM25ContentFilter(RelevantContentFilter):
             user_query (str): User query for filtering (optional).
             bm25_threshold (float): BM25 threshold for filtering (default: 1.0).
             language (str): Language for stemming (default: 'english').
+            use_stemming (bool): Whether to apply stemming (default: True).
         """
         super().__init__(user_query=user_query)
         self.bm25_threshold = bm25_threshold
+        self.use_stemming = use_stemming
         self.priority_tags = {
             "h1": 5.0,
             "h2": 4.0,
@@ -433,7 +435,7 @@ class BM25ContentFilter(RelevantContentFilter):
             "pre": 1.5,
             "th": 1.5,  # Table headers
         }
-        self.stemmer = stemmer(language)
+        self.stemmer = stemmer(language) if use_stemming else None
 
     def filter_content(self, html: str, min_word_threshold: int = None) -> List[str]:
         """
@@ -480,13 +482,19 @@ class BM25ContentFilter(RelevantContentFilter):
         #                 for _, chunk, _, _ in candidates]
         # tokenized_query = [ps.stem(word) for word in query.lower().split()]
 
-        tokenized_corpus = [
-            [self.stemmer.stemWord(word) for word in chunk.lower().split()]
-            for _, chunk, _, _ in candidates
-        ]
-        tokenized_query = [
-            self.stemmer.stemWord(word) for word in query.lower().split()
-        ]
+        if self.use_stemming:
+            tokenized_corpus = [
+                [self.stemmer.stemWord(word) for word in chunk.lower().split()]
+                for _, chunk, _, _ in candidates
+            ]
+            tokenized_query = [
+                self.stemmer.stemWord(word) for word in query.lower().split()
+            ]
+        else:
+            tokenized_corpus = [
+                chunk.lower().split() for _, chunk, _, _ in candidates
+            ]
+            tokenized_query = query.lower().split()
 
         # tokenized_corpus = [[self.stemmer.stemWord(word) for word in tokenize_text(chunk.lower())]
         #            for _, chunk, _, _ in candidates]
@@ -846,8 +854,7 @@ class LLMContentFilter(RelevantContentFilter):
                 },
                 colors={
                     **AsyncLogger.DEFAULT_COLORS,
-                    LogLevel.INFO: Fore.MAGENTA
-                    + Style.DIM,  # Dimmed purple for LLM ops
+                    LogLevel.INFO: LogColor.DIM_MAGENTA  # Dimmed purple for LLM ops
                 },
             )
         else:
@@ -892,7 +899,7 @@ class LLMContentFilter(RelevantContentFilter):
                 "Starting LLM markdown content filtering process",
                 tag="LLM",
                 params={"provider": self.llm_config.provider},
-                colors={"provider": Fore.CYAN},
+                colors={"provider": LogColor.CYAN},
             )
 
         # Cache handling
@@ -929,7 +936,7 @@ class LLMContentFilter(RelevantContentFilter):
                 "LLM markdown: Split content into {chunk_count} chunks",
                 tag="CHUNK",
                 params={"chunk_count": len(html_chunks)},
-                colors={"chunk_count": Fore.YELLOW},
+                colors={"chunk_count": LogColor.YELLOW},
             )
 
         start_time = time.time()
@@ -1038,7 +1045,7 @@ class LLMContentFilter(RelevantContentFilter):
                 "LLM markdown: Completed processing in {time:.2f}s",
                 tag="LLM",
                 params={"time": end_time - start_time},
-                colors={"time": Fore.YELLOW},
+                colors={"time": LogColor.YELLOW},
             )
 
         result = ordered_results if ordered_results else []

crawl4ai/crawlers/google_search/crawler.py

@@ -1,7 +1,7 @@
 from crawl4ai import BrowserConfig, AsyncWebCrawler, CrawlerRunConfig, CacheMode
 from crawl4ai.hub import BaseCrawler
 from crawl4ai.utils import optimize_html, get_home_folder, preprocess_html_for_schema
-from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai import JsonCssExtractionStrategy
 from pathlib import Path
 import json
 import os

crawl4ai/deep_crawling/bff_strategy.py

@@ -11,6 +11,7 @@ from .scorers import URLScorer
 from . import DeepCrawlStrategy
 
 from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+from ..utils import normalize_url_for_deep_crawl
 
 from math import inf as infinity
 
@@ -106,13 +107,14 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
         valid_links = []
         for link in links:
             url = link.get("href")
-            if url in visited:
+            base_url = normalize_url_for_deep_crawl(url, source_url)
+            if base_url in visited:
                 continue
             if not await self.can_process_url(url, new_depth):
                 self.stats.urls_skipped += 1
                 continue
                 
-            valid_links.append(url)
+            valid_links.append(base_url)
             
         # If we have more valid links than capacity, limit them
         if len(valid_links) > remaining_capacity:
@@ -148,6 +150,14 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
                 self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
                 break
                 
+            # Calculate how many more URLs we can process in this batch
+            remaining = self.max_pages - self._pages_crawled
+            batch_size = min(BATCH_SIZE, remaining)
+            if batch_size <= 0:
+                # No more pages to crawl
+                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                break
+                
             batch: List[Tuple[float, int, str, Optional[str]]] = []
             # Retrieve up to BATCH_SIZE items from the priority queue.
             for _ in range(BATCH_SIZE):
@@ -182,6 +192,10 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
                 # Count only successful crawls toward max_pages limit
                 if result.success:
                     self._pages_crawled += 1
+                    # Check if we've reached the limit during batch processing
+                    if self._pages_crawled >= self.max_pages:
+                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                        break  # Exit the generator
                 
                 yield result
                 

crawl4ai/deep_crawling/bfs_strategy.py

@@ -117,7 +117,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
                 self.logger.debug(f"URL {url} skipped: score {score} below threshold {self.score_threshold}")
                 self.stats.urls_skipped += 1
                 continue
-            
+
+            visited.add(base_url)
             valid_links.append((base_url, score))
         
         # If we have more valid links than capacity, sort by score and take the top ones
@@ -156,9 +157,13 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         results: List[CrawlResult] = []
 
         while current_level and not self._cancel_event.is_set():
+            # Check if we've already reached max_pages before starting a new level
+            if self._pages_crawled >= self.max_pages:
+                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                break
+            
             next_level: List[Tuple[str, Optional[str]]] = []
             urls = [url for url, _ in current_level]
-            visited.update(urls)
 
             # Clone the config to disable deep crawling recursion and enforce batch mode.
             batch_config = config.clone(deep_crawl_strategy=None, stream=False)
@@ -221,6 +226,10 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
                 # Count only successful crawls
                 if result.success:
                     self._pages_crawled += 1
+                    # Check if we've reached the limit during batch processing
+                    if self._pages_crawled >= self.max_pages:
+                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                        break  # Exit the generator
                 
                 results_count += 1
                 yield result

crawl4ai/deep_crawling/dfs_strategy.py

@@ -49,6 +49,10 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
                 # Count only successful crawls toward max_pages limit
                 if result.success:
                     self._pages_crawled += 1
+                    # Check if we've reached the limit during batch processing
+                    if self._pages_crawled >= self.max_pages:
+                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                        break  # Exit the generator
                     
                     # Only discover links from successful crawls
                     new_links: List[Tuple[str, Optional[str]]] = []
@@ -94,6 +98,10 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
                 # and only discover links from successful crawls
                 if result.success:
                     self._pages_crawled += 1
+                    # Check if we've reached the limit during batch processing
+                    if self._pages_crawled >= self.max_pages:
+                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+                        break  # Exit the generator
                     
                     new_links: List[Tuple[str, Optional[str]]] = []
                     await self.link_discovery(result, url, depth, visited, new_links, depths)

crawl4ai/deep_crawling/filters.py

@@ -227,10 +227,21 @@ class URLPatternFilter(URLFilter):
         # Prefix check (/foo/*)
         if self._simple_prefixes:
             path = url.split("?")[0]
-            if any(path.startswith(p) for p in self._simple_prefixes):
-                result = True
-                self._update_stats(result)
-                return not result if self._reverse else result
+            # if any(path.startswith(p) for p in self._simple_prefixes):
+            #     result = True
+            #     self._update_stats(result)
+            #     return not result if self._reverse else result
+            ####
+            # Modified the prefix matching logic to ensure path boundary checking:
+            # - Check if the matched prefix is followed by a path separator (`/`), query parameter (`?`), fragment (`#`), or is at the end of the path
+            # - This ensures `/api/` only matches complete path segments, not substrings like `/apiv2/`
+            ####
+            for prefix in self._simple_prefixes:
+                if path.startswith(prefix):
+                    if len(path) == len(prefix) or path[len(prefix)] in ['/', '?', '#']:
+                        result = True
+                        self._update_stats(result)
+                        return not result if self._reverse else result
 
         # Complex patterns
         if self._path_patterns:
@@ -337,6 +348,15 @@ class ContentTypeFilter(URLFilter):
         "sqlite": "application/vnd.sqlite3",
         # Placeholder
         "unknown": "application/octet-stream",  # Fallback for unknown file types
+        # php
+        "php": "application/x-httpd-php",
+        "php3": "application/x-httpd-php",
+        "php4": "application/x-httpd-php",
+        "php5": "application/x-httpd-php",
+        "php7": "application/x-httpd-php",
+        "phtml": "application/x-httpd-php",
+        "phps": "application/x-httpd-php-source",
+
     }
 
     @staticmethod

crawl4ai/docker_client.py

@@ -73,6 +73,8 @@ class Crawl4aiDockerClient:
     def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, 
                        crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]:
         """Prepare request data from configs."""
+        if self._token:
+            self._http_client.headers["Authorization"] = f"Bearer {self._token}"
         return {
             "urls": urls,
             "browser_config": browser_config.dump() if browser_config else {},
@@ -103,8 +105,6 @@ class Crawl4aiDockerClient:
         crawler_config: Optional[CrawlerRunConfig] = None
     ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
         """Execute a crawl operation."""
-        if not self._token:
-            raise Crawl4aiClientError("Authentication required. Call authenticate() first.")
         await self._check_server()
         
         data = self._prepare_request(urls, browser_config, crawler_config)
@@ -140,8 +140,6 @@ class Crawl4aiDockerClient:
 
     async def get_schema(self) -> Dict[str, Any]:
         """Retrieve configuration schemas."""
-        if not self._token:
-            raise Crawl4aiClientError("Authentication required. Call authenticate() first.")
         response = await self._request("GET", "/schema")
         return response.json()
 
@@ -167,4 +165,4 @@ async def main():
         print(schema)
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(main())

crawl4ai/extraction_strategy.py

@@ -1,13 +1,16 @@
 from abc import ABC, abstractmethod
 import inspect
-from typing import Any, List, Dict, Optional
+from typing import Any, List, Dict, Optional, Tuple, Pattern, Union
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import json
 import time
+from enum import IntFlag, auto
 
 from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH, PROMPT_EXTRACT_INFERRED_SCHEMA
 from .config import (
-    DEFAULT_PROVIDER, CHUNK_TOKEN_THRESHOLD,
+    DEFAULT_PROVIDER,
+    DEFAULT_PROVIDER_API_KEY,
+    CHUNK_TOKEN_THRESHOLD,
     OVERLAP_RATE,
     WORD_TOKEN_RATE,
 )
@@ -538,10 +541,15 @@ class LLMExtractionStrategy(ExtractionStrategy):
             api_token: The API token for the provider.
             base_url: The base URL for the API request.
             api_base: The base URL for the API request.
-            extra_args: Additional arguments for the API request, such as temprature, max_tokens, etc.
+            extra_args: Additional arguments for the API request, such as temperature, max_tokens, etc.
         """
         super().__init__( input_format=input_format, **kwargs)
         self.llm_config = llm_config
+        if not self.llm_config:
+            self.llm_config = create_llm_config(
+                provider=DEFAULT_PROVIDER,
+                api_token=os.environ.get(DEFAULT_PROVIDER_API_KEY),
+            )
         self.instruction = instruction
         self.extract_type = extraction_type
         self.schema = schema
@@ -648,11 +656,11 @@ class LLMExtractionStrategy(ExtractionStrategy):
             self.total_usage.total_tokens += usage.total_tokens
 
             try:
-                response = response.choices[0].message.content
+                content = response.choices[0].message.content
                 blocks = None
 
                 if self.force_json_response:
-                    blocks = json.loads(response)
+                    blocks = json.loads(content)
                     if isinstance(blocks, dict):
                         # If it has only one key which calue is list then assign that to blocks, exampled: {"news": [..]}
                         if len(blocks) == 1 and isinstance(list(blocks.values())[0], list):
@@ -665,14 +673,14 @@ class LLMExtractionStrategy(ExtractionStrategy):
                         blocks = blocks
                 else: 
                     # blocks = extract_xml_data(["blocks"], response.choices[0].message.content)["blocks"]
-                    blocks = extract_xml_data(["blocks"], response)["blocks"]
+                    blocks = extract_xml_data(["blocks"], content)["blocks"]
                     blocks = json.loads(blocks)
 
                 for block in blocks:
                     block["error"] = False
             except Exception:
                 parsed, unparsed = split_and_parse_json_objects(
-                    response
+                    response.choices[0].message.content
                 )
                 blocks = parsed
                 if unparsed:
@@ -1160,7 +1168,11 @@ In this scenario, use your best judgment to generate the schema. You need to exa
         elif not query and not target_json_example:
             user_message["content"] += """IMPORTANT: Since we neither have a query nor an example, it is crucial to rely solely on the HTML content provided. Leverage your expertise to determine the schema based on the repetitive patterns observed in the content."""
         
-        user_message["content"] += """IMPORTANT: Ensure your schema remains reliable by avoiding selectors that appear to generate dynamically and are not dependable. You want a reliable schema, as it consistently returns the same data even after many page reloads.
+        user_message["content"] += """IMPORTANT: 
+        0/ Ensure your schema remains reliable by avoiding selectors that appear to generate dynamically and are not dependable. You want a reliable schema, as it consistently returns the same data even after many page reloads.
+        1/ DO NOT USE use base64 kind of classes, they are temporary and not reliable.
+        2/ Every selector must refer to only one unique element. You should ensure your selector points to a single element and is unique to the place that contains the information. You have to use available techniques based on CSS or XPATH requested schema to make sure your selector is unique and also not fragile, meaning if we reload the page now or in the future, the selector should remain reliable.
+        3/ Do not use Regex as much as possible.
 
         Analyze the HTML and generate a JSON schema that follows the specified format. Only output valid JSON schema, nothing else.
         """
@@ -1661,3 +1673,303 @@ class JsonXPathExtractionStrategy(JsonElementExtractionStrategy):
     def _get_element_attribute(self, element, attribute: str):
         return element.get(attribute)
 
+"""
+RegexExtractionStrategy
+Fast, zero-LLM extraction of common entities via regular expressions.
+"""
+
+_CTRL = {c: rf"\x{ord(c):02x}" for c in map(chr, range(32)) if c not in "\t\n\r"}
+
+_WB_FIX = re.compile(r"\x08")               # stray back-space   →   word-boundary
+_NEEDS_ESCAPE = re.compile(r"(?<!\\)\\(?![\\u])")   # lone backslash
+
+def _sanitize_schema(schema: Dict[str, str]) -> Dict[str, str]:
+    """Fix common JSON-escape goofs coming from LLMs or manual edits."""
+    safe = {}
+    for label, pat in schema.items():
+        # 1️⃣ replace accidental control chars (inc. the infamous back-space)
+        pat = _WB_FIX.sub(r"\\b", pat).translate(_CTRL)
+
+        # 2️⃣ double any single backslash that JSON kept single
+        pat = _NEEDS_ESCAPE.sub(r"\\\\", pat)
+
+        # 3️⃣ quick sanity compile
+        try:
+            re.compile(pat)
+        except re.error as e:
+            raise ValueError(f"Regex for '{label}' won’t compile after fix: {e}") from None
+
+        safe[label] = pat
+    return safe
+
+
+class RegexExtractionStrategy(ExtractionStrategy):
+    """
+    A lean strategy that finds e-mails, phones, URLs, dates, money, etc.,
+    using nothing but pre-compiled regular expressions.
+
+    Extraction returns::
+
+        {
+            "url":   "<page-url>",
+            "label": "<pattern-label>",
+            "value": "<matched-string>",
+            "span":  [start, end]
+        }
+
+    Only `generate_schema()` touches an LLM, extraction itself is pure Python.
+    """
+
+    # -------------------------------------------------------------- #
+    # Built-in patterns exposed as IntFlag so callers can bit-OR them
+    # -------------------------------------------------------------- #
+    class _B(IntFlag):
+        EMAIL           = auto()
+        PHONE_INTL      = auto()
+        PHONE_US        = auto()
+        URL             = auto()
+        IPV4            = auto()
+        IPV6            = auto()
+        UUID            = auto()
+        CURRENCY        = auto()
+        PERCENTAGE      = auto()
+        NUMBER          = auto()
+        DATE_ISO        = auto()
+        DATE_US         = auto()
+        TIME_24H        = auto()
+        POSTAL_US       = auto()
+        POSTAL_UK       = auto()
+        HTML_COLOR_HEX  = auto()
+        TWITTER_HANDLE  = auto()
+        HASHTAG         = auto()
+        MAC_ADDR        = auto()
+        IBAN            = auto()
+        CREDIT_CARD     = auto()
+        NOTHING         = auto()
+        ALL             = (
+            EMAIL | PHONE_INTL | PHONE_US | URL | IPV4 | IPV6 | UUID
+            | CURRENCY | PERCENTAGE | NUMBER | DATE_ISO | DATE_US | TIME_24H
+            | POSTAL_US | POSTAL_UK | HTML_COLOR_HEX | TWITTER_HANDLE
+            | HASHTAG | MAC_ADDR | IBAN | CREDIT_CARD
+        )
+
+    # user-friendly aliases  (RegexExtractionStrategy.Email, .IPv4, …)
+    Email          = _B.EMAIL
+    PhoneIntl      = _B.PHONE_INTL
+    PhoneUS        = _B.PHONE_US
+    Url            = _B.URL
+    IPv4           = _B.IPV4
+    IPv6           = _B.IPV6
+    Uuid           = _B.UUID
+    Currency       = _B.CURRENCY
+    Percentage     = _B.PERCENTAGE
+    Number         = _B.NUMBER
+    DateIso        = _B.DATE_ISO
+    DateUS         = _B.DATE_US
+    Time24h        = _B.TIME_24H
+    PostalUS       = _B.POSTAL_US
+    PostalUK       = _B.POSTAL_UK
+    HexColor       = _B.HTML_COLOR_HEX
+    TwitterHandle  = _B.TWITTER_HANDLE
+    Hashtag        = _B.HASHTAG
+    MacAddr        = _B.MAC_ADDR
+    Iban           = _B.IBAN
+    CreditCard     = _B.CREDIT_CARD
+    All            = _B.ALL
+    Nothing        = _B(0)  # no patterns
+
+    # ------------------------------------------------------------------ #
+    # Built-in pattern catalog
+    # ------------------------------------------------------------------ #
+    DEFAULT_PATTERNS: Dict[str, str] = {
+        # Communication
+        "email":           r"[\w.+-]+@[\w-]+\.[\w.-]+",
+        "phone_intl":      r"\+?\d[\d .()-]{7,}\d",
+        "phone_us":        r"\(?\d{3}\)?[ -. ]?\d{3}[ -. ]?\d{4}",
+        # Web
+        "url":             r"https?://[^\s\"'<>]+",
+        "ipv4":            r"(?:\d{1,3}\.){3}\d{1,3}",
+        "ipv6":            r"[A-F0-9]{1,4}(?::[A-F0-9]{1,4}){7}",
+        # IDs
+        "uuid":            r"[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}",
+        # Money / numbers
+        "currency":        r"(?:USD|EUR|RM|\$|€|£)\s?\d+(?:[.,]\d{2})?",
+        "percentage":      r"\d+(?:\.\d+)?%",
+        "number":          r"\b\d{1,3}(?:[,.\s]\d{3})*(?:\.\d+)?\b",
+        # Dates / Times
+        "date_iso":        r"\d{4}-\d{2}-\d{2}",
+        "date_us":         r"\d{1,2}/\d{1,2}/\d{2,4}",
+        "time_24h":        r"\b(?:[01]?\d|2[0-3]):[0-5]\d(?:[:.][0-5]\d)?\b",
+        # Misc
+        "postal_us":       r"\b\d{5}(?:-\d{4})?\b",
+        "postal_uk":       r"\b[A-Z]{1,2}\d[A-Z\d]? ?\d[A-Z]{2}\b",
+        "html_color_hex":  r"#[0-9A-Fa-f]{6}\b",
+        "twitter_handle":  r"@[\w]{1,15}",
+        "hashtag":         r"#[\w-]+",
+        "mac_addr":        r"(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}",
+        "iban":            r"[A-Z]{2}\d{2}[A-Z0-9]{11,30}",
+        "credit_card":     r"\b(?:4\d{12}(?:\d{3})?|5[1-5]\d{14}|3[47]\d{13}|6(?:011|5\d{2})\d{12})\b",
+    }
+
+    _FLAGS = re.IGNORECASE | re.MULTILINE
+    _UNWANTED_PROPS = {
+        "provider": "Use llm_config instead",
+        "api_token": "Use llm_config instead",
+    }
+
+    # ------------------------------------------------------------------ #
+    # Construction
+    # ------------------------------------------------------------------ #
+    def __init__(
+        self,
+        pattern: "_B" = _B.NOTHING,
+        *,
+        custom: Optional[Union[Dict[str, str], List[Tuple[str, str]]]] = None,
+        input_format: str = "fit_html",
+        **kwargs,
+    ) -> None:
+        """
+        Args:
+            patterns: Custom patterns overriding or extending defaults.
+                      Dict[label, regex] or list[tuple(label, regex)].
+            input_format: "html", "markdown" or "text".
+            **kwargs: Forwarded to ExtractionStrategy.
+        """
+        super().__init__(input_format=input_format, **kwargs)
+
+        # 1️⃣  take only the requested built-ins
+        merged: Dict[str, str] = {
+            key: rx
+            for key, rx in self.DEFAULT_PATTERNS.items()
+            if getattr(self._B, key.upper()).value & pattern
+        }
+
+        # 2️⃣  apply user overrides / additions
+        if custom:
+            if isinstance(custom, dict):
+                merged.update(custom)
+            else:  # iterable of (label, regex)
+                merged.update({lbl: rx for lbl, rx in custom})
+
+        self._compiled: Dict[str, Pattern] = {
+            lbl: re.compile(rx, self._FLAGS) for lbl, rx in merged.items()
+        }
+
+    # ------------------------------------------------------------------ #
+    # Extraction
+    # ------------------------------------------------------------------ #
+    def extract(self, url: str, content: str, *q, **kw) -> List[Dict[str, Any]]:
+        # text = self._plain_text(html)
+        out: List[Dict[str, Any]] = []
+
+        for label, cre in self._compiled.items():
+            for m in cre.finditer(content):
+                out.append(
+                    {
+                        "url": url,
+                        "label": label,
+                        "value": m.group(0),
+                        "span": [m.start(), m.end()],
+                    }
+                )
+        return out
+
+    # ------------------------------------------------------------------ #
+    # Helpers
+    # ------------------------------------------------------------------ #
+    def _plain_text(self, content: str) -> str:
+        if self.input_format == "text":
+            return content
+        return BeautifulSoup(content, "lxml").get_text(" ", strip=True)
+
+    # ------------------------------------------------------------------ #
+    # LLM-assisted pattern generator
+    # ------------------------------------------------------------------ #
+    # ------------------------------------------------------------------ #
+    # LLM-assisted one-off pattern builder
+    # ------------------------------------------------------------------ #
+    @staticmethod
+    def generate_pattern(
+        label: str,
+        html: str,
+        *,
+        query: Optional[str] = None,
+        examples: Optional[List[str]] = None,
+        llm_config: Optional[LLMConfig] = None,
+        **kwargs,
+    ) -> Dict[str, str]:
+        """
+        Ask an LLM for a single page-specific regex and return
+            {label: pattern}   ── ready for RegexExtractionStrategy(custom=…)
+        """
+
+        # ── guard deprecated kwargs
+        for k in RegexExtractionStrategy._UNWANTED_PROPS:
+            if k in kwargs:
+                raise AttributeError(
+                    f"{k} is deprecated, {RegexExtractionStrategy._UNWANTED_PROPS[k]}"
+                )
+
+        # ── default LLM config
+        if llm_config is None:
+            llm_config = create_llm_config()
+
+        # ── system prompt – hardened
+        system_msg = (
+            "You are an expert Python-regex engineer.\n"
+            f"Return **one** JSON object whose single key is exactly \"{label}\", "
+            "and whose value is a raw-string regex pattern that works with "
+            "the standard `re` module in Python.\n\n"
+            "Strict rules (obey every bullet):\n"
+            "• If a *user query* is supplied, treat it as the precise semantic target and optimise the "
+            "  pattern to capture ONLY text that answers that query. If the query conflicts with the "
+            "  sample HTML, the HTML wins.\n"
+            "• Tailor the pattern to the *sample HTML* – reproduce its exact punctuation, spacing, "
+            "  symbols, capitalisation, etc. Do **NOT** invent a generic form.\n"
+            "• Keep it minimal and fast: avoid unnecessary capturing, prefer non-capturing `(?: … )`, "
+            "  and guard against catastrophic backtracking.\n"
+            "• Anchor with `^`, `$`, or `\\b` only when it genuinely improves precision.\n"
+            "• Use inline flags like `(?i)` when needed; no verbose flag comments.\n"
+            "• Output must be valid JSON – no markdown, code fences, comments, or extra keys.\n"
+            "• The regex value must be a Python string literal: **double every backslash** "
+            "(e.g. `\\\\b`, `\\\\d`, `\\\\\\\\`).\n\n"
+            "Example valid output:\n"
+            f"{{\"{label}\": \"(?:RM|rm)\\\\s?\\\\d{{1,3}}(?:,\\\\d{{3}})*(?:\\\\.\\\\d{{2}})?\"}}"
+        )
+
+        # ── user message: cropped HTML + optional hints
+        user_parts = ["```html", html[:5000], "```"]  # protect token budget
+        if query:
+            user_parts.append(f"\n\n## Query\n{query.strip()}")
+        if examples:
+            user_parts.append("## Examples\n" + "\n".join(examples[:20]))
+        user_msg = "\n\n".join(user_parts)
+
+        # ── LLM call (with retry/backoff)
+        resp = perform_completion_with_backoff(
+            provider=llm_config.provider,
+            prompt_with_variables="\n\n".join([system_msg, user_msg]),
+            json_response=True,
+            api_token=llm_config.api_token,
+            base_url=llm_config.base_url,
+            extra_args=kwargs,
+        )
+
+        # ── clean & load JSON (fix common escape mistakes *before* json.loads)
+        raw = resp.choices[0].message.content
+        raw = raw.replace("\x08", "\\b")                     # stray back-space → \b
+        raw = re.sub(r'(?<!\\)\\(?![\\u"])', r"\\\\", raw)   # lone \ → \\
+
+        try:
+            pattern_dict = json.loads(raw)
+        except Exception as exc:
+            raise ValueError(f"LLM did not return valid JSON: {raw}") from exc
+
+        # quick sanity-compile
+        for lbl, pat in pattern_dict.items():
+            try:
+                re.compile(pat)
+            except re.error as e:
+                raise ValueError(f"Invalid regex for '{lbl}': {e}") from None
+
+        return pattern_dict

crawl4ai/install.py

@@ -40,10 +40,25 @@ def setup_home_directory():
             f.write("")
 
 def post_install():
-    """Run all post-installation tasks"""
+    """
+    Run all post-installation tasks.
+    Checks CRAWL4AI_MODE environment variable. If set to 'api',
+    skips Playwright browser installation.
+    """
     logger.info("Running post-installation setup...", tag="INIT")
     setup_home_directory()
-    install_playwright()
+
+    # Check environment variable to conditionally skip Playwright install
+    run_mode = os.getenv('CRAWL4AI_MODE')
+    if run_mode == 'api':
+        logger.warning(
+            "CRAWL4AI_MODE=api detected. Skipping Playwright browser installation.",
+            tag="SETUP"
+        )
+    else:
+        # Proceed with installation only if mode is not 'api'
+        install_playwright()
+
     run_migration()
     # TODO: Will be added in the future
     # setup_builtin_browser()
@@ -104,6 +119,32 @@ def install_playwright():
         logger.warning(
             f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
         )
+    
+    # Install Patchright browsers for undetected browser support
+    logger.info("Installing Patchright browsers for undetected mode...", tag="INIT")
+    try:
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                "patchright",
+                "install",
+                "--with-deps",
+                "--force",
+                "chromium",
+            ]
+        )
+        logger.success(
+            "Patchright installation completed successfully.", tag="COMPLETE"
+        )
+    except subprocess.CalledProcessError:
+        logger.warning(
+            f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."
+        )
+    except Exception:
+        logger.warning(
+            f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."
+        )
 
 
 def run_migration():

crawl4ai/js_snippet/remove_overlay_elements.js

@@ -115,5 +115,6 @@ async () => {
     document.body.style.overflow = "auto";
 
     // Wait a bit for any animations to complete
-    await new Promise((resolve) => setTimeout(resolve, 100));
+    document.body.scrollIntoView(false);
+    await new Promise((resolve) => setTimeout(resolve, 50));
 };

crawl4ai/legacy/web_crawler.py

@@ -11,7 +11,7 @@ from .extraction_strategy import *
 from .crawler_strategy import *
 from typing import List
 from concurrent.futures import ThreadPoolExecutor
-from .content_scraping_strategy import WebScrapingStrategy
+from ..content_scraping_strategy import LXMLWebScrapingStrategy as WebScrapingStrategy
 from .config import *
 import warnings
 import json

crawl4ai/link_preview.py

@@ -0,0 +1,395 @@
+"""
+Link Extractor for Crawl4AI
+
+Extracts head content from links discovered during crawling using URLSeeder's
+efficient parallel processing and caching infrastructure.
+"""
+
+import asyncio
+import fnmatch
+from typing import Dict, List, Optional, Any
+from .async_logger import AsyncLogger
+from .async_url_seeder import AsyncUrlSeeder
+from .async_configs import SeedingConfig, CrawlerRunConfig
+from .models import Links, Link
+from .utils import calculate_total_score
+
+
+class LinkPreview:
+    """
+    Extracts head content from links using URLSeeder's parallel processing infrastructure.
+    
+    This class provides intelligent link filtering and head content extraction with:
+    - Pattern-based inclusion/exclusion filtering
+    - Parallel processing with configurable concurrency
+    - Caching for performance
+    - BM25 relevance scoring
+    - Memory-safe processing for large link sets
+    """
+    
+    def __init__(self, logger: Optional[AsyncLogger] = None):
+        """
+        Initialize the LinkPreview.
+        
+        Args:
+            logger: Optional logger instance for recording events
+        """
+        self.logger = logger
+        self.seeder: Optional[AsyncUrlSeeder] = None
+        self._owns_seeder = False
+    
+    async def __aenter__(self):
+        """Async context manager entry."""
+        await self.start()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self.close()
+    
+    async def start(self):
+        """Initialize the URLSeeder instance."""
+        if not self.seeder:
+            self.seeder = AsyncUrlSeeder(logger=self.logger)
+            await self.seeder.__aenter__()
+            self._owns_seeder = True
+    
+    async def close(self):
+        """Clean up resources."""
+        if self.seeder and self._owns_seeder:
+            await self.seeder.__aexit__(None, None, None)
+            self.seeder = None
+            self._owns_seeder = False
+    
+    def _log(self, level: str, message: str, tag: str = "LINK_EXTRACT", **kwargs):
+        """Helper method to safely log messages."""
+        if self.logger:
+            log_method = getattr(self.logger, level, None)
+            if log_method:
+                log_method(message=message, tag=tag, params=kwargs.get('params', {}))
+    
+    async def extract_link_heads(
+        self, 
+        links: Links, 
+        config: CrawlerRunConfig
+    ) -> Links:
+        """
+        Extract head content for filtered links and attach to Link objects.
+        
+        Args:
+            links: Links object containing internal and external links
+            config: CrawlerRunConfig with link_preview_config settings
+            
+        Returns:
+            Links object with head_data attached to filtered Link objects
+        """
+        link_config = config.link_preview_config
+        
+        # Ensure seeder is initialized
+        await self.start()
+        
+        # Filter links based on configuration
+        filtered_urls = self._filter_links(links, link_config)
+        
+        if not filtered_urls:
+            self._log("info", "No links matched filtering criteria")
+            return links
+        
+        self._log("info", "Extracting head content for {count} filtered links",
+                  params={"count": len(filtered_urls)})
+        
+        # Extract head content using URLSeeder
+        head_results = await self._extract_heads_parallel(filtered_urls, link_config)
+        
+        # Merge results back into Link objects
+        updated_links = self._merge_head_data(links, head_results, config)
+        
+        self._log("info", "Completed head extraction for links, {success} successful",
+                  params={"success": len([r for r in head_results if r.get("status") == "valid"])})
+        
+        return updated_links
+    
+    def _filter_links(self, links: Links, link_config: Dict[str, Any]) -> List[str]:
+        """
+        Filter links based on configuration parameters.
+        
+        Args:
+            links: Links object containing internal and external links
+            link_config: Configuration dictionary for link extraction
+            
+        Returns:
+            List of filtered URL strings
+        """
+        filtered_urls = []
+        
+        # Include internal links if configured
+        if link_config.include_internal:
+            filtered_urls.extend([link.href for link in links.internal if link.href])
+            self._log("debug", "Added {count} internal links",
+                      params={"count": len(links.internal)})
+        
+        # Include external links if configured
+        if link_config.include_external:
+            filtered_urls.extend([link.href for link in links.external if link.href])
+            self._log("debug", "Added {count} external links",
+                      params={"count": len(links.external)})
+        
+        # Apply include patterns
+        include_patterns = link_config.include_patterns
+        if include_patterns:
+            filtered_urls = [
+                url for url in filtered_urls
+                if any(fnmatch.fnmatch(url, pattern) for pattern in include_patterns)
+            ]
+            self._log("debug", "After include patterns: {count} links remain",
+                      params={"count": len(filtered_urls)})
+        
+        # Apply exclude patterns
+        exclude_patterns = link_config.exclude_patterns
+        if exclude_patterns:
+            filtered_urls = [
+                url for url in filtered_urls
+                if not any(fnmatch.fnmatch(url, pattern) for pattern in exclude_patterns)
+            ]
+            self._log("debug", "After exclude patterns: {count} links remain",
+                      params={"count": len(filtered_urls)})
+        
+        # Limit number of links
+        max_links = link_config.max_links
+        if max_links > 0 and len(filtered_urls) > max_links:
+            filtered_urls = filtered_urls[:max_links]
+            self._log("debug", "Limited to {max_links} links",
+                      params={"max_links": max_links})
+        
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_urls = []
+        for url in filtered_urls:
+            if url not in seen:
+                seen.add(url)
+                unique_urls.append(url)
+        
+        self._log("debug", "Final filtered URLs: {count} unique links",
+                  params={"count": len(unique_urls)})
+        
+        return unique_urls
+    
+    async def _extract_heads_parallel(
+        self, 
+        urls: List[str], 
+        link_config: Dict[str, Any]
+    ) -> List[Dict[str, Any]]:
+        """
+        Extract head content for URLs using URLSeeder's parallel processing.
+        
+        Args:
+            urls: List of URLs to process
+            link_config: Configuration dictionary for link extraction
+            
+        Returns:
+            List of dictionaries with url, status, head_data, and optional relevance_score
+        """
+        verbose = link_config.verbose
+        concurrency = link_config.concurrency
+        
+        if verbose:
+            self._log("info", "Starting batch processing: {total} links with {concurrency} concurrent workers",
+                      params={"total": len(urls), "concurrency": concurrency})
+        
+        # Create SeedingConfig for URLSeeder
+        seeding_config = SeedingConfig(
+            extract_head=True,
+            concurrency=concurrency,
+            hits_per_sec=getattr(link_config, 'hits_per_sec', None),
+            query=link_config.query,
+            score_threshold=link_config.score_threshold,
+            scoring_method="bm25" if link_config.query else None,
+            verbose=verbose
+        )
+        
+        # Use URLSeeder's extract_head_for_urls method with progress tracking
+        if verbose:
+            # Create a wrapper to track progress
+            results = await self._extract_with_progress(urls, seeding_config, link_config)
+        else:
+            results = await self.seeder.extract_head_for_urls(
+                urls=urls,
+                config=seeding_config,
+                concurrency=concurrency,
+                timeout=link_config.timeout
+            )
+        
+        return results
+    
+    async def _extract_with_progress(
+        self, 
+        urls: List[str], 
+        seeding_config: SeedingConfig, 
+        link_config: Dict[str, Any]
+    ) -> List[Dict[str, Any]]:
+        """Extract head content with progress reporting."""
+        
+        total_urls = len(urls)
+        concurrency = link_config.concurrency
+        batch_size = max(1, total_urls // 10)  # Report progress every 10%
+        
+        # Process URLs and track progress
+        completed = 0
+        successful = 0
+        failed = 0
+        
+        # Create a custom progress tracking version
+        # We'll modify URLSeeder's method to include progress callbacks
+        
+        # For now, let's use the existing method and report at the end
+        # In a production version, we would modify URLSeeder to accept progress callbacks
+        
+        self._log("info", "Processing links in batches...")
+        
+        # Use existing method
+        results = await self.seeder.extract_head_for_urls(
+            urls=urls,
+            config=seeding_config,
+            concurrency=concurrency,
+            timeout=link_config.timeout
+        )
+        
+        # Count results
+        for result in results:
+            completed += 1
+            if result.get("status") == "valid":
+                successful += 1
+            else:
+                failed += 1
+        
+        # Final progress report
+        self._log("info", "Batch processing completed: {completed}/{total} processed, {successful} successful, {failed} failed",
+                  params={
+                      "completed": completed,
+                      "total": total_urls,
+                      "successful": successful,
+                      "failed": failed
+                  })
+        
+        return results
+    
+    def _merge_head_data(
+        self, 
+        original_links: Links, 
+        head_results: List[Dict[str, Any]],
+        config: CrawlerRunConfig
+    ) -> Links:
+        """
+        Merge head extraction results back into Link objects.
+        
+        Args:
+            original_links: Original Links object
+            head_results: Results from head extraction
+            
+        Returns:
+            Links object with head_data attached to matching links
+        """
+        # Create URL to head_data mapping
+        url_to_head_data = {}
+        for result in head_results:
+            url = result.get("url")
+            if url:
+                url_to_head_data[url] = {
+                    "head_data": result.get("head_data", {}),
+                    "status": result.get("status", "unknown"),
+                    "error": result.get("error"),
+                    "relevance_score": result.get("relevance_score")
+                }
+        
+        # Update internal links
+        updated_internal = []
+        for link in original_links.internal:
+            if link.href in url_to_head_data:
+                head_info = url_to_head_data[link.href]
+                # Create new Link object with head data and scoring
+                contextual_score = head_info.get("relevance_score")
+                
+                updated_link = Link(
+                    href=link.href,
+                    text=link.text,
+                    title=link.title,
+                    base_domain=link.base_domain,
+                    head_data=head_info["head_data"],
+                    head_extraction_status=head_info["status"],
+                    head_extraction_error=head_info.get("error"),
+                    intrinsic_score=getattr(link, 'intrinsic_score', None),
+                    contextual_score=contextual_score
+                )
+                
+                # Add relevance score to head_data for backward compatibility
+                if contextual_score is not None:
+                    updated_link.head_data = updated_link.head_data or {}
+                    updated_link.head_data["relevance_score"] = contextual_score
+                
+                # Calculate total score combining intrinsic and contextual scores
+                updated_link.total_score = calculate_total_score(
+                    intrinsic_score=updated_link.intrinsic_score,
+                    contextual_score=updated_link.contextual_score,
+                    score_links_enabled=getattr(config, 'score_links', False),
+                    query_provided=bool(config.link_preview_config.query)
+                )
+                
+                updated_internal.append(updated_link)
+            else:
+                # Keep original link unchanged
+                updated_internal.append(link)
+        
+        # Update external links
+        updated_external = []
+        for link in original_links.external:
+            if link.href in url_to_head_data:
+                head_info = url_to_head_data[link.href]
+                # Create new Link object with head data and scoring
+                contextual_score = head_info.get("relevance_score")
+                
+                updated_link = Link(
+                    href=link.href,
+                    text=link.text,
+                    title=link.title,
+                    base_domain=link.base_domain,
+                    head_data=head_info["head_data"],
+                    head_extraction_status=head_info["status"],
+                    head_extraction_error=head_info.get("error"),
+                    intrinsic_score=getattr(link, 'intrinsic_score', None),
+                    contextual_score=contextual_score
+                )
+                
+                # Add relevance score to head_data for backward compatibility
+                if contextual_score is not None:
+                    updated_link.head_data = updated_link.head_data or {}
+                    updated_link.head_data["relevance_score"] = contextual_score
+                
+                # Calculate total score combining intrinsic and contextual scores
+                updated_link.total_score = calculate_total_score(
+                    intrinsic_score=updated_link.intrinsic_score,
+                    contextual_score=updated_link.contextual_score,
+                    score_links_enabled=getattr(config, 'score_links', False),
+                    query_provided=bool(config.link_preview_config.query)
+                )
+                
+                updated_external.append(updated_link)
+            else:
+                # Keep original link unchanged
+                updated_external.append(link)
+        
+        # Sort links by relevance score if available
+        if any(hasattr(link, 'head_data') and link.head_data and 'relevance_score' in link.head_data 
+               for link in updated_internal + updated_external):
+            
+            def get_relevance_score(link):
+                if hasattr(link, 'head_data') and link.head_data and 'relevance_score' in link.head_data:
+                    return link.head_data['relevance_score']
+                return 0.0
+            
+            updated_internal.sort(key=get_relevance_score, reverse=True)
+            updated_external.sort(key=get_relevance_score, reverse=True)
+        
+        return Links(
+            internal=updated_internal,
+            external=updated_external
+        )

crawl4ai/markdown_generation_strategy.py

@@ -31,22 +31,24 @@ class MarkdownGenerationStrategy(ABC):
         content_filter: Optional[RelevantContentFilter] = None,
         options: Optional[Dict[str, Any]] = None,
         verbose: bool = False,
+        content_source: str = "cleaned_html",
     ):
         self.content_filter = content_filter
         self.options = options or {}
         self.verbose = verbose
+        self.content_source = content_source
 
     @abstractmethod
     def generate_markdown(
         self,
-        cleaned_html: str,
+        input_html: str,
         base_url: str = "",
         html2text_options: Optional[Dict[str, Any]] = None,
         content_filter: Optional[RelevantContentFilter] = None,
         citations: bool = True,
         **kwargs,
     ) -> MarkdownGenerationResult:
-        """Generate markdown from cleaned HTML."""
+        """Generate markdown from the selected input HTML."""
         pass
 
 
@@ -63,6 +65,7 @@ class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
     Args:
         content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
         options (Optional[Dict[str, Any]]): Additional options for markdown generation. Defaults to None.
+        content_source (str): Source of content to generate markdown from. Options: "cleaned_html", "raw_html", "fit_html". Defaults to "cleaned_html".
 
     Returns:
         MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
@@ -72,8 +75,9 @@ class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
         self,
         content_filter: Optional[RelevantContentFilter] = None,
         options: Optional[Dict[str, Any]] = None,
+        content_source: str = "cleaned_html",
     ):
-        super().__init__(content_filter, options)
+        super().__init__(content_filter, options, verbose=False, content_source=content_source)
 
     def convert_links_to_citations(
         self, markdown: str, base_url: str = ""
@@ -143,7 +147,7 @@ class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
 
     def generate_markdown(
         self,
-        cleaned_html: str,
+        input_html: str,
         base_url: str = "",
         html2text_options: Optional[Dict[str, Any]] = None,
         options: Optional[Dict[str, Any]] = None,
@@ -152,16 +156,16 @@ class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
         **kwargs,
     ) -> MarkdownGenerationResult:
         """
-        Generate markdown with citations from cleaned HTML.
+        Generate markdown with citations from the provided input HTML.
 
         How it works:
-        1. Generate raw markdown from cleaned HTML.
+        1. Generate raw markdown from the input HTML.
         2. Convert links to citations.
         3. Generate fit markdown if content filter is provided.
         4. Return MarkdownGenerationResult.
 
         Args:
-            cleaned_html (str): Cleaned HTML content.
+            input_html (str): The HTML content to process (selected based on content_source).
             base_url (str): Base URL for URL joins.
             html2text_options (Optional[Dict[str, Any]]): HTML2Text options.
             options (Optional[Dict[str, Any]]): Additional options for markdown generation.
@@ -196,14 +200,14 @@ class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
             h.update_params(**default_options)
 
             # Ensure we have valid input
-            if not cleaned_html:
-                cleaned_html = ""
-            elif not isinstance(cleaned_html, str):
-                cleaned_html = str(cleaned_html)
+            if not input_html:
+                input_html = ""
+            elif not isinstance(input_html, str):
+                input_html = str(input_html)
 
             # Generate raw markdown
             try:
-                raw_markdown = h.handle(cleaned_html)
+                raw_markdown = h.handle(input_html)
             except Exception as e:
                 raw_markdown = f"Error converting HTML to markdown: {str(e)}"
 
@@ -228,7 +232,7 @@ class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
             if content_filter or self.content_filter:
                 try:
                     content_filter = content_filter or self.content_filter
-                    filtered_html = content_filter.filter_content(cleaned_html)
+                    filtered_html = content_filter.filter_content(input_html)
                     filtered_html = "\n".join(
                         "<div>{}</div>".format(s) for s in filtered_html
                     )

crawl4ai/memory_utils.py

@@ -0,0 +1,79 @@
+import psutil
+import platform
+import subprocess
+from typing import Tuple
+
+
+def get_true_available_memory_gb() -> float:
+    """Get truly available memory including inactive pages (cross-platform)"""
+    vm = psutil.virtual_memory()
+
+    if platform.system() == 'Darwin':  # macOS
+        # On macOS, we need to include inactive memory too
+        try:
+            # Use vm_stat to get accurate values
+            result = subprocess.run(['vm_stat'], capture_output=True, text=True)
+            lines = result.stdout.split('\n')
+
+            page_size = 16384  # macOS page size
+            pages = {}
+
+            for line in lines:
+                if 'Pages free:' in line:
+                    pages['free'] = int(line.split()[-1].rstrip('.'))
+                elif 'Pages inactive:' in line:
+                    pages['inactive'] = int(line.split()[-1].rstrip('.'))
+                elif 'Pages speculative:' in line:
+                    pages['speculative'] = int(line.split()[-1].rstrip('.'))
+                elif 'Pages purgeable:' in line:
+                    pages['purgeable'] = int(line.split()[-1].rstrip('.'))
+
+            # Calculate total available (free + inactive + speculative + purgeable)
+            total_available_pages = (
+                pages.get('free', 0) + 
+                pages.get('inactive', 0) + 
+                pages.get('speculative', 0) + 
+                pages.get('purgeable', 0)
+            )
+            available_gb = (total_available_pages * page_size) / (1024**3)
+
+            return available_gb
+        except:
+            # Fallback to psutil
+            return vm.available / (1024**3)
+    else:
+        # For Windows and Linux, psutil.available is accurate
+        return vm.available / (1024**3)
+
+
+def get_true_memory_usage_percent() -> float:
+    """
+    Get memory usage percentage that accounts for platform differences.
+    
+    Returns:
+        float: Memory usage percentage (0-100)
+    """
+    vm = psutil.virtual_memory()
+    total_gb = vm.total / (1024**3)
+    available_gb = get_true_available_memory_gb()
+    
+    # Calculate used percentage based on truly available memory
+    used_percent = 100.0 * (total_gb - available_gb) / total_gb
+    
+    # Ensure it's within valid range
+    return max(0.0, min(100.0, used_percent))
+
+
+def get_memory_stats() -> Tuple[float, float, float]:
+    """
+    Get comprehensive memory statistics.
+    
+    Returns:
+        Tuple[float, float, float]: (used_percent, available_gb, total_gb)
+    """
+    vm = psutil.virtual_memory()
+    total_gb = vm.total / (1024**3)
+    available_gb = get_true_available_memory_gb()
+    used_percent = get_true_memory_usage_percent()
+    
+    return used_percent, available_gb, total_gb

crawl4ai/models.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, HttpUrl, PrivateAttr, ConfigDict
+from pydantic import BaseModel, HttpUrl, PrivateAttr, Field
 from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
 from typing import AsyncGenerator
 from typing import Generic, TypeVar
@@ -95,15 +95,7 @@ class UrlModel(BaseModel):
     url: HttpUrl
     forced: bool = False
 
-class MarkdownGenerationResult(BaseModel):
-    raw_markdown: str
-    markdown_with_citations: str
-    references_markdown: str
-    fit_markdown: Optional[str] = None
-    fit_html: Optional[str] = None
 
-    def __str__(self):
-        return self.raw_markdown
 
 @dataclass
 class TraversalStats:
@@ -124,9 +116,20 @@ class DispatchResult(BaseModel):
     end_time: Union[datetime, float]
     error_message: str = ""
 
+class MarkdownGenerationResult(BaseModel):
+    raw_markdown: str
+    markdown_with_citations: str
+    references_markdown: str
+    fit_markdown: Optional[str] = None
+    fit_html: Optional[str] = None
+
+    def __str__(self):
+        return self.raw_markdown
+    
 class CrawlResult(BaseModel):
     url: str
     html: str
+    fit_html: Optional[str] = None
     success: bool
     cleaned_html: Optional[str] = None
     media: Dict[str, List[Dict]] = {}
@@ -135,6 +138,7 @@ class CrawlResult(BaseModel):
     js_execution_result: Optional[Dict[str, Any]] = None
     screenshot: Optional[str] = None
     pdf: Optional[bytes] = None
+    mhtml: Optional[str] = None
     _markdown: Optional[MarkdownGenerationResult] = PrivateAttr(default=None)
     extracted_content: Optional[str] = None
     metadata: Optional[dict] = None
@@ -145,10 +149,12 @@ class CrawlResult(BaseModel):
     ssl_certificate: Optional[SSLCertificate] = None
     dispatch_result: Optional[DispatchResult] = None
     redirected_url: Optional[str] = None
+    network_requests: Optional[List[Dict[str, Any]]] = None
+    console_messages: Optional[List[Dict[str, Any]]] = None
+    tables: List[Dict] = Field(default_factory=list)  # NEW – [{headers,rows,caption,summary}]
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-    # class Config:
-    #     arbitrary_types_allowed = True
+    class Config:
+        arbitrary_types_allowed = True
 
 # NOTE: The StringCompatibleMarkdown class, custom __init__ method, property getters/setters,
 # and model_dump override all exist to support a smooth transition from markdown as a string
@@ -308,14 +314,16 @@ class AsyncCrawlResponse(BaseModel):
     status_code: int
     screenshot: Optional[str] = None
     pdf_data: Optional[bytes] = None
+    mhtml_data: Optional[str] = None
     get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
     downloaded_files: Optional[List[str]] = None
     ssl_certificate: Optional[SSLCertificate] = None
     redirected_url: Optional[str] = None
+    network_requests: Optional[List[Dict[str, Any]]] = None
+    console_messages: Optional[List[Dict[str, Any]]] = None
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-    # class Config:
-    #     arbitrary_types_allowed = True
+    class Config:
+        arbitrary_types_allowed = True
 
 ###############################
 # Scraping Models
@@ -337,6 +345,12 @@ class Link(BaseModel):
     text: Optional[str] = ""
     title: Optional[str] = ""
     base_domain: Optional[str] = ""
+    head_data: Optional[Dict[str, Any]] = None  # Head metadata extracted from link target
+    head_extraction_status: Optional[str] = None  # "success", "failed", "skipped"
+    head_extraction_error: Optional[str] = None  # Error message if extraction failed
+    intrinsic_score: Optional[float] = None  # Quality score based on URL structure, text, and context
+    contextual_score: Optional[float] = None  # BM25 relevance score based on query and head content
+    total_score: Optional[float] = None  # Combined score from intrinsic and contextual scores
 
 
 class Media(BaseModel):

crawl4ai/pipeline/__init__.py

@@ -1,6 +0,0 @@
-"""Pipeline module providing high-level crawling functionality."""
-
-from .pipeline import Pipeline, create_pipeline
-from .crawler import Crawler
-
-__all__ = ["Pipeline", "create_pipeline", "Crawler"]

crawl4ai/pipeline/crawler.py

@@ -1,406 +0,0 @@
-"""Crawler utility class for simplified crawling operations.
-
-This module provides a high-level utility class for crawling web pages
-with support for both single and multiple URL processing.
-"""
-
-import asyncio
-from typing import Dict, List, Optional, Tuple, Union, Callable
-
-from crawl4ai.models import CrawlResultContainer, CrawlResult
-from crawl4ai.pipeline.pipeline import create_pipeline
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.browser.browser_hub import BrowserHub
-
-# Type definitions
-UrlList = List[str]
-UrlBatch = Tuple[List[str], CrawlerRunConfig]
-UrlFullBatch = Tuple[List[str], BrowserConfig, CrawlerRunConfig]
-BatchType = Union[UrlList, UrlBatch, UrlFullBatch]
-ProgressCallback = Callable[[str, str, Optional[CrawlResultContainer]], None]
-RetryStrategy = Callable[[str, int, Exception], Tuple[bool, float]]
-
-class Crawler:
-    """High-level utility class for crawling web pages.
-    
-    This class provides simplified methods for crawling both single URLs
-    and batches of URLs, with parallel processing capabilities.
-    """
-    
-    @classmethod
-    async def crawl(
-        cls, 
-        urls: Union[str, List[str]],
-        browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None,
-        browser_hub: Optional[BrowserHub] = None,
-        logger: Optional[AsyncLogger] = None,
-        max_retries: int = 0,
-        retry_delay: float = 1.0,
-        use_new_loop: bool = True  # By default use a new loop for safety
-    ) -> Union[CrawlResultContainer, Dict[str, CrawlResultContainer]]:
-        """Crawl one or more URLs with the specified configurations.
-        
-        Args:
-            urls: Single URL or list of URLs to crawl
-            browser_config: Optional browser configuration
-            crawler_config: Optional crawler run configuration
-            browser_hub: Optional shared browser hub
-            logger: Optional logger instance
-            max_retries: Maximum number of retries for failed requests
-            retry_delay: Delay between retries in seconds
-            
-        Returns:
-            For a single URL: CrawlResultContainer with crawl results
-            For multiple URLs: Dict mapping URLs to their CrawlResultContainer results
-        """
-        # Handle single URL case
-        if isinstance(urls, str):
-            return await cls._crawl_single_url(
-                urls,
-                browser_config,
-                crawler_config,
-                browser_hub,
-                logger,
-                max_retries,
-                retry_delay,
-                use_new_loop
-            )
-        
-        # Handle multiple URLs case (sequential processing)
-        results = {}
-        for url in urls:
-            results[url] = await cls._crawl_single_url(
-                url,
-                browser_config,
-                crawler_config,
-                browser_hub,
-                logger,
-                max_retries,
-                retry_delay,
-                use_new_loop
-            )
-        
-        return results
-    
-    @classmethod
-    async def _crawl_single_url(
-        cls,
-        url: str,
-        browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None,
-        browser_hub: Optional[BrowserHub] = None,
-        logger: Optional[AsyncLogger] = None,
-        max_retries: int = 0,
-        retry_delay: float = 1.0,
-        use_new_loop: bool = False
-    ) -> CrawlResultContainer:
-        """Internal method to crawl a single URL with retry logic."""
-        # Create a logger if none provided
-        if logger is None:
-            logger = AsyncLogger(verbose=True)
-            
-        # Create or use the provided crawler config
-        if crawler_config is None:
-            crawler_config = CrawlerRunConfig()
-            
-        attempts = 0
-        last_error = None
-        
-        # For testing purposes, each crawler gets a new event loop to avoid conflicts
-        # This is especially important in test suites where multiple tests run in sequence
-        if use_new_loop:
-            old_loop = asyncio.get_event_loop()
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            
-        while attempts <= max_retries:
-            try:
-                # Create a pipeline
-                pipeline_args = {}
-                if browser_config:
-                    pipeline_args["browser_config"] = browser_config
-                if browser_hub:
-                    pipeline_args["browser_hub"] = browser_hub
-                if logger:
-                    pipeline_args["logger"] = logger
-                    
-                pipeline = await create_pipeline(**pipeline_args)
-                
-                # Perform the crawl
-                result = await pipeline.crawl(url=url, config=crawler_config)
-                
-                # Close the pipeline if we created it (not using a shared hub)
-                if not browser_hub:
-                    await pipeline.close()
-                    
-                # Restore the original event loop if we created a new one
-                if use_new_loop:
-                    asyncio.set_event_loop(old_loop)
-                    loop.close()
-                    
-                return result
-                
-            except Exception as e:
-                last_error = e
-                attempts += 1
-                
-                if attempts <= max_retries:
-                    logger.warning(
-                        message="Crawl attempt {attempt} failed for {url}: {error}. Retrying in {delay}s...",
-                        tag="RETRY",
-                        params={
-                            "attempt": attempts,
-                            "url": url,
-                            "error": str(e),
-                            "delay": retry_delay
-                        }
-                    )
-                    await asyncio.sleep(retry_delay)
-                else:
-                    logger.error(
-                        message="All {attempts} crawl attempts failed for {url}: {error}",
-                        tag="FAILED",
-                        params={
-                            "attempts": attempts,
-                            "url": url,
-                            "error": str(e)
-                        }
-                    )
-        
-        # If we get here, all attempts failed
-        result = CrawlResultContainer(
-            CrawlResult(
-                url=url,
-                html="",
-                success=False,
-                error_message=f"All {attempts} crawl attempts failed: {str(last_error)}"
-            )
-        )
-        
-        # Restore the original event loop if we created a new one
-        if use_new_loop:
-            asyncio.set_event_loop(old_loop)
-            loop.close()
-            
-        return result
-    
-    @classmethod
-    async def parallel_crawl(
-        cls,
-        url_batches: Union[List[str], List[Union[UrlBatch, UrlFullBatch]]],
-        browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None,
-        browser_hub: Optional[BrowserHub] = None,
-        logger: Optional[AsyncLogger] = None,
-        concurrency: int = 5,
-        max_retries: int = 0,
-        retry_delay: float = 1.0,
-        retry_strategy: Optional[RetryStrategy] = None,
-        progress_callback: Optional[ProgressCallback] = None,
-        use_new_loop: bool = True  # By default use a new loop for safety
-    ) -> Dict[str, CrawlResultContainer]:
-        """Crawl multiple URLs in parallel with concurrency control.
-        
-        Args:
-            url_batches: List of URLs or list of URL batches with configurations
-            browser_config: Default browser configuration (used if not in batch)
-            crawler_config: Default crawler configuration (used if not in batch)
-            browser_hub: Optional shared browser hub for resource efficiency
-            logger: Optional logger instance
-            concurrency: Maximum number of concurrent crawls
-            max_retries: Maximum number of retries for failed requests
-            retry_delay: Delay between retries in seconds
-            retry_strategy: Optional custom retry strategy function
-            progress_callback: Optional callback for progress reporting
-            
-        Returns:
-            Dict mapping URLs to their CrawlResultContainer results
-        """
-        # Create a logger if none provided
-        if logger is None:
-            logger = AsyncLogger(verbose=True)
-        
-        # For testing purposes, each crawler gets a new event loop to avoid conflicts
-        # This is especially important in test suites where multiple tests run in sequence
-        if use_new_loop:
-            old_loop = asyncio.get_event_loop()
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            
-        # Process batches to consistent format
-        processed_batches = cls._process_url_batches(
-            url_batches, browser_config, crawler_config
-        )
-        
-        # Initialize results dictionary
-        results = {}
-        
-        # Create semaphore for concurrency control
-        semaphore = asyncio.Semaphore(concurrency)
-        
-        # Create shared browser hub if not provided
-        shared_hub = browser_hub
-        if not shared_hub:
-            shared_hub = await BrowserHub.get_browser_manager(
-                config=browser_config or BrowserConfig(),
-                logger=logger,
-                max_browsers_per_config=concurrency,
-                max_pages_per_browser=1,
-                initial_pool_size=min(concurrency, 3)  # Start with a reasonable number
-            )
-        
-        try:
-            # Create worker function for each URL
-            async def process_url(url, b_config, c_config):
-                async with semaphore:
-                    # Report start if callback provided
-                    if progress_callback:
-                        await progress_callback("started", url)
-                    
-                    attempts = 0
-                    last_error = None
-                    
-                    while attempts <= max_retries:
-                        try:
-                            # Create a pipeline using the shared hub
-                            pipeline = await create_pipeline(
-                                browser_config=b_config,
-                                browser_hub=shared_hub,
-                                logger=logger
-                            )
-                            
-                            # Perform the crawl
-                            result = await pipeline.crawl(url=url, config=c_config)
-                            
-                            # Report completion if callback provided
-                            if progress_callback:
-                                await progress_callback("completed", url, result)
-                                
-                            return url, result
-                            
-                        except Exception as e:
-                            last_error = e
-                            attempts += 1
-                            
-                            # Determine if we should retry and with what delay
-                            should_retry = attempts <= max_retries
-                            delay = retry_delay
-                            
-                            # Use custom retry strategy if provided
-                            if retry_strategy and should_retry:
-                                try:
-                                    should_retry, delay = await retry_strategy(url, attempts, e)
-                                except Exception as strategy_error:
-                                    logger.error(
-                                        message="Error in retry strategy: {error}",
-                                        tag="RETRY",
-                                        params={"error": str(strategy_error)}
-                                    )
-                            
-                            if should_retry:
-                                logger.warning(
-                                    message="Crawl attempt {attempt} failed for {url}: {error}. Retrying in {delay}s...",
-                                    tag="RETRY",
-                                    params={
-                                        "attempt": attempts,
-                                        "url": url,
-                                        "error": str(e),
-                                        "delay": delay
-                                    }
-                                )
-                                await asyncio.sleep(delay)
-                            else:
-                                logger.error(
-                                    message="All {attempts} crawl attempts failed for {url}: {error}",
-                                    tag="FAILED",
-                                    params={
-                                        "attempts": attempts,
-                                        "url": url,
-                                        "error": str(e)
-                                    }
-                                )
-                                break
-                    
-                    # If we get here, all attempts failed
-                    error_result = CrawlResultContainer(
-                        CrawlResult(
-                            url=url,
-                            html="",
-                            success=False,
-                            error_message=f"All {attempts} crawl attempts failed: {str(last_error)}"
-                        )
-                    )
-                    
-                    # Report completion with error if callback provided
-                    if progress_callback:
-                        await progress_callback("completed", url, error_result)
-                        
-                    return url, error_result
-            
-            # Create tasks for all URLs
-            tasks = []
-            for urls, b_config, c_config in processed_batches:
-                for url in urls:
-                    tasks.append(process_url(url, b_config, c_config))
-            
-            # Run all tasks and collect results
-            for completed_task in asyncio.as_completed(tasks):
-                url, result = await completed_task
-                results[url] = result
-                
-            return results
-            
-        finally:
-            # Clean up the hub only if we created it
-            if not browser_hub and shared_hub:
-                await shared_hub.close()
-                
-            # Restore the original event loop if we created a new one
-            if use_new_loop:
-                asyncio.set_event_loop(old_loop)
-                loop.close()
-    
-    @classmethod
-    def _process_url_batches(
-        cls,
-        url_batches: Union[List[str], List[Union[UrlBatch, UrlFullBatch]]],
-        default_browser_config: Optional[BrowserConfig],
-        default_crawler_config: Optional[CrawlerRunConfig]
-    ) -> List[Tuple[List[str], BrowserConfig, CrawlerRunConfig]]:
-        """Process URL batches into a consistent format.
-        
-        Converts various input formats into a consistent list of
-        (urls, browser_config, crawler_config) tuples.
-        """
-        processed_batches = []
-        
-        # Handle case where input is just a list of URLs
-        if all(isinstance(item, str) for item in url_batches):
-            urls = url_batches
-            browser_config = default_browser_config or BrowserConfig()
-            crawler_config = default_crawler_config or CrawlerRunConfig()
-            processed_batches.append((urls, browser_config, crawler_config))
-            return processed_batches
-        
-        # Process each batch
-        for batch in url_batches:
-            # Handle case: (urls, crawler_config)
-            if len(batch) == 2 and isinstance(batch[1], CrawlerRunConfig):
-                urls, c_config = batch
-                b_config = default_browser_config or BrowserConfig()
-                processed_batches.append((urls, b_config, c_config))
-                
-            # Handle case: (urls, browser_config, crawler_config)
-            elif len(batch) == 3 and isinstance(batch[1], BrowserConfig) and isinstance(batch[2], CrawlerRunConfig):
-                processed_batches.append(batch)
-                
-            # Fallback for unknown formats - assume it's just a list of URLs
-            else:
-                urls = batch
-                browser_config = default_browser_config or BrowserConfig()
-                crawler_config = default_crawler_config or CrawlerRunConfig()
-                processed_batches.append((urls, browser_config, crawler_config))
-                
-        return processed_batches

crawl4ai/pipeline/middlewares.py

@@ -1,702 +0,0 @@
-import time
-import sys
-from typing import Dict, Any, List
-import json
-
-from crawl4ai.models import (
-    CrawlResult,
-    MarkdownGenerationResult,
-    ScrapingResult,
-    CrawlResultContainer,
-)
-from crawl4ai.async_database import async_db_manager
-from crawl4ai.cache_context import CacheMode, CacheContext
-from crawl4ai.utils import (
-    sanitize_input_encode,
-    InvalidCSSSelectorError,
-    fast_format_html,
-    create_box_message,
-    get_error_context,
-)
-
-
-async def initialize_context_middleware(context: Dict[str, Any]) -> int:
-    """Initialize the context with basic configuration and validation"""
-    url = context.get("url")
-    config = context.get("config")
-    
-    if not isinstance(url, str) or not url:
-        context["error_message"] = "Invalid URL, make sure the URL is a non-empty string"
-        return 0
-    
-    # Default to ENABLED if no cache mode specified
-    if config.cache_mode is None:
-        config.cache_mode = CacheMode.ENABLED
-    
-    # Create cache context
-    context["cache_context"] = CacheContext(url, config.cache_mode, False)
-    context["start_time"] = time.perf_counter()
-    
-    return 1
-
-# middlewares.py additions
-
-async def browser_hub_middleware(context: Dict[str, Any]) -> int:
-    """
-    Initialize or connect to a Browser-Hub and add it to the pipeline context.
-    
-    This middleware handles browser hub initialization for all three scenarios:
-    1. Default configuration when nothing is specified
-    2. Custom configuration when browser_config is provided
-    3. Connection to existing hub when browser_hub_connection is provided
-    
-    Args:
-        context: The pipeline context dictionary
-        
-    Returns:
-        int: 1 for success, 0 for failure
-    """
-    from crawl4ai.browser.browser_hub import BrowserHub
-    
-    try:
-        # Get configuration from context
-        browser_config = context.get("browser_config")
-        browser_hub_id = context.get("browser_hub_id")
-        browser_hub_connection = context.get("browser_hub_connection")
-        logger = context.get("logger")
-        
-        # If we already have a browser hub in context, use it
-        if context.get("browser_hub"):
-            return 1
-        
-        # Get or create Browser-Hub
-        browser_hub = await BrowserHub.get_browser_manager(
-            config=browser_config,
-            hub_id=browser_hub_id,
-            connection_info=browser_hub_connection,
-            logger=logger
-        )
-        
-        # Add to context
-        context["browser_hub"] = browser_hub
-        return 1
-    except Exception as e:
-        context["error_message"] = f"Failed to initialize browser hub: {str(e)}"
-        return 0
-
-
-async def fetch_content_middleware(context: Dict[str, Any]) -> int:
-    """
-    Fetch content from the web using the browser hub.
-    
-    This middleware uses the browser hub to get pages for crawling,
-    and properly releases them back to the pool when done.
-    
-    Args:
-        context: The pipeline context dictionary
-        
-    Returns:
-        int: 1 for success, 0 for failure
-    """
-    url = context.get("url")
-    config = context.get("config")
-    browser_hub = context.get("browser_hub")
-    logger = context.get("logger")
-    
-    # Skip if using cached result
-    if context.get("cached_result") and context.get("html"):
-        return 1
-    
-    try:
-        # Create crawler strategy without initializing its browser manager
-        from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
-        
-        crawler_strategy = AsyncPlaywrightCrawlerStrategy(
-            browser_config=browser_hub.config if browser_hub else None,
-            logger=logger
-        )
-        
-        # Replace the browser manager with our shared instance
-        crawler_strategy.browser_manager = browser_hub
-        
-        # Perform crawl without trying to initialize the browser
-        # The crawler will use the provided browser_manager to get pages
-        async_response = await crawler_strategy.crawl(url, config=config)
-        
-        # Store results in context
-        context["html"] = async_response.html
-        context["screenshot_data"] = async_response.screenshot
-        context["pdf_data"] = async_response.pdf_data
-        context["js_execution_result"] = async_response.js_execution_result
-        context["async_response"] = async_response
-        
-        return 1
-    except Exception as e:
-        context["error_message"] = f"Error fetching content: {str(e)}"
-        return 0
-
-
-async def check_cache_middleware(context: Dict[str, Any]) -> int:
-    """Check if there's a cached result and load it if available"""
-    url = context.get("url")
-    config = context.get("config")
-    cache_context = context.get("cache_context")
-    logger = context.get("logger")
-    
-    # Initialize variables
-    context["cached_result"] = None
-    context["html"] = None
-    context["extracted_content"] = None
-    context["screenshot_data"] = None
-    context["pdf_data"] = None
-    
-    # Try to get cached result if appropriate
-    if cache_context.should_read():
-        cached_result = await async_db_manager.aget_cached_url(url)
-        context["cached_result"] = cached_result
-        
-        if cached_result:
-            html = sanitize_input_encode(cached_result.html)
-            extracted_content = sanitize_input_encode(cached_result.extracted_content or "")
-            extracted_content = None if not extracted_content or extracted_content == "[]" else extracted_content
-            
-            # If screenshot is requested but its not in cache, then set cache_result to None
-            screenshot_data = cached_result.screenshot
-            pdf_data = cached_result.pdf
-            
-            if config.screenshot and not screenshot_data:
-                context["cached_result"] = None
-            
-            if config.pdf and not pdf_data:
-                context["cached_result"] = None
-            
-            context["html"] = html
-            context["extracted_content"] = extracted_content
-            context["screenshot_data"] = screenshot_data
-            context["pdf_data"] = pdf_data
-            
-            logger.url_status(
-                url=cache_context.display_url,
-                success=bool(html),
-                timing=time.perf_counter() - context["start_time"],
-                tag="FETCH",
-            )
-    
-    return 1
-
-
-async def configure_proxy_middleware(context: Dict[str, Any]) -> int:
-    """Configure proxy if a proxy rotation strategy is available"""
-    config = context.get("config")
-    logger = context.get("logger")
-    
-    # Skip if using cached result
-    if context.get("cached_result") and context.get("html"):
-        return 1
-    
-    # Update proxy configuration from rotation strategy if available
-    if config and config.proxy_rotation_strategy:
-        next_proxy = await config.proxy_rotation_strategy.get_next_proxy()
-        if next_proxy:
-            logger.info(
-                message="Switch proxy: {proxy}",
-                tag="PROXY",
-                params={"proxy": next_proxy.server},
-            )
-            config.proxy_config = next_proxy
-    
-    return 1
-
-
-async def check_robots_txt_middleware(context: Dict[str, Any]) -> int:
-    """Check if the URL is allowed by robots.txt if enabled"""
-    url = context.get("url")
-    config = context.get("config")
-    browser_config = context.get("browser_config")
-    robots_parser = context.get("robots_parser")
-    
-    # Skip if using cached result
-    if context.get("cached_result") and context.get("html"):
-        return 1
-    
-    # Check robots.txt if enabled
-    if config and config.check_robots_txt:
-        if not await robots_parser.can_fetch(url, browser_config.user_agent):
-            context["crawl_result"] = CrawlResult(
-                url=url,
-                html="",
-                success=False,
-                status_code=403,
-                error_message="Access denied by robots.txt",
-                response_headers={"X-Robots-Status": "Blocked by robots.txt"}
-            )
-            return 0
-    
-    return 1
-
-
-async def fetch_content_middleware_(context: Dict[str, Any]) -> int:
-    """Fetch content from the web using the crawler strategy"""
-    url = context.get("url")
-    config = context.get("config")
-    crawler_strategy = context.get("crawler_strategy")
-    logger = context.get("logger")
-    
-    # Skip if using cached result
-    if context.get("cached_result") and context.get("html"):
-        return 1
-    
-    try:
-        t1 = time.perf_counter()
-        
-        if config.user_agent:
-            crawler_strategy.update_user_agent(config.user_agent)
-        
-        # Call CrawlerStrategy.crawl
-        async_response = await crawler_strategy.crawl(url, config=config)
-        
-        html = sanitize_input_encode(async_response.html)
-        screenshot_data = async_response.screenshot
-        pdf_data = async_response.pdf_data
-        js_execution_result = async_response.js_execution_result
-        
-        t2 = time.perf_counter()
-        logger.url_status(
-            url=context["cache_context"].display_url,
-            success=bool(html),
-            timing=t2 - t1,
-            tag="FETCH",
-        )
-        
-        context["html"] = html
-        context["screenshot_data"] = screenshot_data
-        context["pdf_data"] = pdf_data
-        context["js_execution_result"] = js_execution_result
-        context["async_response"] = async_response
-        
-        return 1
-    except Exception as e:
-        context["error_message"] = f"Error fetching content: {str(e)}"
-        return 0
-
-
-async def scrape_content_middleware(context: Dict[str, Any]) -> int:
-    """Apply scraping strategy to extract content"""
-    url = context.get("url")
-    html = context.get("html")
-    config = context.get("config")
-    extracted_content = context.get("extracted_content")
-    logger = context.get("logger")
-    
-    # Skip if already have a crawl result
-    if context.get("crawl_result"):
-        return 1
-    
-    try:
-        _url = url if not context.get("is_raw_html", False) else "Raw HTML"
-        t1 = time.perf_counter()
-        
-        # Get scraping strategy and ensure it has a logger
-        scraping_strategy = config.scraping_strategy
-        if not scraping_strategy.logger:
-            scraping_strategy.logger = logger
-        
-        # Process HTML content
-        params = config.__dict__.copy()
-        params.pop("url", None)
-        # Add keys from kwargs to params that don't exist in params
-        kwargs = context.get("kwargs", {})
-        params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
-        
-        # Scraping Strategy Execution
-        result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
-        
-        if result is None:
-            raise ValueError(f"Process HTML, Failed to extract content from the website: {url}")
-        
-        # Extract results - handle both dict and ScrapingResult
-        if isinstance(result, dict):
-            cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
-            media = result.get("media", {})
-            links = result.get("links", {})
-            metadata = result.get("metadata", {})
-        else:
-            cleaned_html = sanitize_input_encode(result.cleaned_html)
-            media = result.media.model_dump()
-            links = result.links.model_dump()
-            metadata = result.metadata
-        
-        context["cleaned_html"] = cleaned_html
-        context["media"] = media
-        context["links"] = links
-        context["metadata"] = metadata
-        
-        # Log processing completion
-        logger.info(
-            message="{url:.50}... | Time: {timing}s",
-            tag="SCRAPE",
-            params={
-                "url": _url,
-                "timing": int((time.perf_counter() - t1) * 1000) / 1000,
-            },
-        )
-        
-        return 1
-    except InvalidCSSSelectorError as e:
-        context["error_message"] = str(e)
-        return 0
-    except Exception as e:
-        context["error_message"] = f"Process HTML, Failed to extract content from the website: {url}, error: {str(e)}"
-        return 0
-
-
-async def generate_markdown_middleware(context: Dict[str, Any]) -> int:
-    """Generate markdown from cleaned HTML"""
-    url = context.get("url")
-    cleaned_html = context.get("cleaned_html")
-    config = context.get("config")
-    
-    # Skip if already have a crawl result
-    if context.get("crawl_result"):
-        return 1
-    
-    # Generate Markdown
-    markdown_generator = config.markdown_generator
-    
-    markdown_result: MarkdownGenerationResult = markdown_generator.generate_markdown(
-        cleaned_html=cleaned_html,
-        base_url=url,
-    )
-    
-    context["markdown_result"] = markdown_result
-    
-    return 1
-
-
-async def extract_structured_content_middleware(context: Dict[str, Any]) -> int:
-    """Extract structured content using extraction strategy"""
-    url = context.get("url")
-    extracted_content = context.get("extracted_content")
-    config = context.get("config")
-    markdown_result = context.get("markdown_result")
-    cleaned_html = context.get("cleaned_html")
-    logger = context.get("logger")
-    
-    # Skip if already have a crawl result or extracted content
-    if context.get("crawl_result") or bool(extracted_content):
-        return 1
-    
-    from crawl4ai.chunking_strategy import IdentityChunking
-    from crawl4ai.extraction_strategy import NoExtractionStrategy
-    
-    if config.extraction_strategy and not isinstance(config.extraction_strategy, NoExtractionStrategy):
-        t1 = time.perf_counter()
-        _url = url if not context.get("is_raw_html", False) else "Raw HTML"
-        
-        # Choose content based on input_format
-        content_format = config.extraction_strategy.input_format
-        if content_format == "fit_markdown" and not markdown_result.fit_markdown:
-            logger.warning(
-                message="Fit markdown requested but not available. Falling back to raw markdown.",
-                tag="EXTRACT",
-                params={"url": _url},
-            )
-            content_format = "markdown"
-        
-        content = {
-            "markdown": markdown_result.raw_markdown,
-            "html": context.get("html"),
-            "cleaned_html": cleaned_html,
-            "fit_markdown": markdown_result.fit_markdown,
-        }.get(content_format, markdown_result.raw_markdown)
-        
-        # Use IdentityChunking for HTML input, otherwise use provided chunking strategy
-        chunking = (
-            IdentityChunking()
-            if content_format in ["html", "cleaned_html"]
-            else config.chunking_strategy
-        )
-        sections = chunking.chunk(content)
-        extracted_content = config.extraction_strategy.run(url, sections)
-        extracted_content = json.dumps(
-            extracted_content, indent=4, default=str, ensure_ascii=False
-        )
-        
-        context["extracted_content"] = extracted_content
-        
-        # Log extraction completion
-        logger.info(
-            message="Completed for {url:.50}... | Time: {timing}s",
-            tag="EXTRACT",
-            params={"url": _url, "timing": time.perf_counter() - t1},
-        )
-    
-    return 1
-
-
-async def format_html_middleware(context: Dict[str, Any]) -> int:
-    """Format HTML if prettify is enabled"""
-    config = context.get("config")
-    cleaned_html = context.get("cleaned_html")
-    
-    # Skip if already have a crawl result
-    if context.get("crawl_result"):
-        return 1
-    
-    # Apply HTML formatting if requested
-    if config.prettiify and cleaned_html:
-        context["cleaned_html"] = fast_format_html(cleaned_html)
-    
-    return 1
-
-
-async def write_cache_middleware(context: Dict[str, Any]) -> int:
-    """Write result to cache if appropriate"""
-    cache_context = context.get("cache_context")
-    cached_result = context.get("cached_result")
-    
-    # Skip if already have a crawl result or not using cache
-    if context.get("crawl_result") or not cache_context.should_write() or bool(cached_result):
-        return 1
-    
-    # We'll create the CrawlResult in build_result_middleware and cache it there
-    # to avoid creating it twice
-    
-    return 1
-
-
-async def build_result_middleware(context: Dict[str, Any]) -> int:
-    """Build the final CrawlResult object"""
-    url = context.get("url")
-    html = context.get("html", "")
-    cache_context = context.get("cache_context")
-    cached_result = context.get("cached_result")
-    config = context.get("config")
-    logger = context.get("logger")
-    
-    # If we already have a crawl result (from an earlier middleware like robots.txt check)
-    if context.get("crawl_result"):
-        result = context["crawl_result"]
-        context["final_result"] = CrawlResultContainer(result)
-        return 1
-    
-    # If we have a cached result
-    if cached_result and html:
-        logger.success(
-            message="{url:.50}... | Status: {status} | Total: {timing}",
-            tag="COMPLETE",
-            params={
-                "url": cache_context.display_url,
-                "status": True,
-                "timing": f"{time.perf_counter() - context['start_time']:.2f}s",
-            },
-            colors={"status": "green", "timing": "yellow"},
-        )
-        
-        cached_result.success = bool(html)
-        cached_result.session_id = getattr(config, "session_id", None)
-        cached_result.redirected_url = cached_result.redirected_url or url
-        context["final_result"] = CrawlResultContainer(cached_result)
-        return 1
-    
-    # Build a new result
-    try:
-        # Get all necessary components from context
-        cleaned_html = context.get("cleaned_html", "")
-        markdown_result = context.get("markdown_result")
-        media = context.get("media", {})
-        links = context.get("links", {})
-        metadata = context.get("metadata", {})
-        screenshot_data = context.get("screenshot_data")
-        pdf_data = context.get("pdf_data")
-        extracted_content = context.get("extracted_content")
-        async_response = context.get("async_response")
-        
-        # Create the CrawlResult
-        crawl_result = CrawlResult(
-            url=url,
-            html=html,
-            cleaned_html=cleaned_html,
-            markdown=markdown_result,
-            media=media,
-            links=links,
-            metadata=metadata,
-            screenshot=screenshot_data,
-            pdf=pdf_data,
-            extracted_content=extracted_content,
-            success=bool(html),
-            error_message="",
-        )
-        
-        # Add response details if available
-        if async_response:
-            crawl_result.status_code = async_response.status_code
-            crawl_result.redirected_url = async_response.redirected_url or url
-            crawl_result.response_headers = async_response.response_headers
-            crawl_result.downloaded_files = async_response.downloaded_files
-            crawl_result.js_execution_result = context.get("js_execution_result")
-            crawl_result.ssl_certificate = async_response.ssl_certificate
-        
-        crawl_result.session_id = getattr(config, "session_id", None)
-        
-        # Log completion
-        logger.success(
-            message="{url:.50}... | Status: {status} | Total: {timing}",
-            tag="COMPLETE",
-            params={
-                "url": cache_context.display_url,
-                "status": crawl_result.success,
-                "timing": f"{time.perf_counter() - context['start_time']:.2f}s",
-            },
-            colors={
-                "status": "green" if crawl_result.success else "red",
-                "timing": "yellow",
-            },
-        )
-        
-        # Update cache if appropriate
-        if cache_context.should_write() and not bool(cached_result):
-            await async_db_manager.acache_url(crawl_result)
-        
-        context["final_result"] = CrawlResultContainer(crawl_result)
-        return 1
-    except Exception as e:
-        error_context = get_error_context(sys.exc_info())
-        
-        error_message = (
-            f"Unexpected error in build_result at line {error_context['line_no']} "
-            f"in {error_context['function']} ({error_context['filename']}):\n"
-            f"Error: {str(e)}\n\n"
-            f"Code context:\n{error_context['code_context']}"
-        )
-        
-        logger.error_status(
-            url=url,
-            error=create_box_message(error_message, type="error"),
-            tag="ERROR",
-        )
-        
-        context["final_result"] = CrawlResultContainer(
-            CrawlResult(
-                url=url, html="", success=False, error_message=error_message
-            )
-        )
-        return 1
-
-
-async def handle_error_middleware(context: Dict[str, Any]) -> Dict[str, Any]:
-    """Error handler middleware"""
-    url = context.get("url", "")
-    error_message = context.get("error_message", "Unknown error")
-    logger = context.get("logger")
-    
-    # Log the error
-    if logger:
-        logger.error_status(
-            url=url,
-            error=create_box_message(error_message, type="error"),
-            tag="ERROR",
-        )
-    
-    # Create a failure result
-    context["final_result"] = CrawlResultContainer(
-        CrawlResult(
-            url=url, html="", success=False, error_message=error_message
-        )
-    )
-    
-    return context
-
-
-# Custom middlewares as requested
-
-async def sentiment_analysis_middleware(context: Dict[str, Any]) -> int:
-    """Analyze sentiment of generated markdown using TextBlob"""
-    from textblob import TextBlob
-    
-    markdown_result = context.get("markdown_result")
-    
-    # Skip if no markdown or already failed
-    if not markdown_result or not context.get("success", True):
-        return 1
-    
-    try:
-        # Get raw markdown text
-        raw_markdown = markdown_result.raw_markdown
-        
-        # Analyze sentiment
-        blob = TextBlob(raw_markdown)
-        sentiment = blob.sentiment
-        
-        # Add sentiment to context
-        context["sentiment_analysis"] = {
-            "polarity": sentiment.polarity,  # -1.0 to 1.0 (negative to positive)
-            "subjectivity": sentiment.subjectivity,  # 0.0 to 1.0 (objective to subjective)
-            "classification": "positive" if sentiment.polarity > 0.1 else 
-                             "negative" if sentiment.polarity < -0.1 else "neutral"
-        }
-        
-        return 1
-    except Exception as e:
-        # Don't fail the pipeline on sentiment analysis failure
-        context["sentiment_analysis_error"] = str(e)
-        return 1
-
-
-async def log_timing_middleware(context: Dict[str, Any], name: str) -> int:
-    """Log timing information for a specific point in the pipeline"""
-    context[f"_timing_mark_{name}"] = time.perf_counter()
-    
-    # Calculate duration if we have a start time
-    start_key = f"_timing_start_{name}"
-    if start_key in context:
-        duration = context[f"_timing_mark_{name}"] - context[start_key]
-        context[f"_timing_duration_{name}"] = duration
-        
-        # Log the timing if we have a logger
-        logger = context.get("logger")
-        if logger:
-            logger.info(
-                message="{name} completed in {duration:.2f}s",
-                tag="TIMING",
-                params={"name": name, "duration": duration},
-            )
-    
-    return 1
-
-
-async def validate_url_middleware(context: Dict[str, Any], patterns: List[str]) -> int:
-    """Validate URL against glob patterns"""
-    import fnmatch
-    url = context.get("url", "")
-    
-    # If no patterns provided, allow all
-    if not patterns:
-        return 1
-    
-    # Check if URL matches any of the allowed patterns
-    for pattern in patterns:
-        if fnmatch.fnmatch(url, pattern):
-            return 1
-    
-    # If we get here, URL didn't match any patterns
-    context["error_message"] = f"URL '{url}' does not match any allowed patterns"
-    return 0
-
-
-# Update the default middleware list function
-def create_default_middleware_list():
-    """Return the default list of middleware functions for the pipeline."""
-    return [
-        initialize_context_middleware,
-        check_cache_middleware,
-        browser_hub_middleware,  # Add browser hub middleware before fetch_content
-        configure_proxy_middleware,
-        check_robots_txt_middleware,
-        fetch_content_middleware,
-        scrape_content_middleware,
-        generate_markdown_middleware,
-        extract_structured_content_middleware,
-        format_html_middleware,
-        build_result_middleware
-    ]

crawl4ai/pipeline/pipeline.py

@@ -1,297 +0,0 @@
-
-import time
-import asyncio
-from typing import Callable, Dict, List, Any, Optional, Awaitable, Union, TypedDict, Tuple, Coroutine
-
-from .middlewares import create_default_middleware_list, handle_error_middleware
-from crawl4ai.models import CrawlResultContainer, CrawlResult
-from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy, AsyncPlaywrightCrawlerStrategy
-from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
-from crawl4ai.async_logger import AsyncLogger
-
-
-class CrawlSpec(TypedDict, total=False):
-    """Specification for a single crawl operation in batch_crawl."""
-    url: str
-    config: Optional[CrawlerRunConfig]
-    browser_config: Optional[BrowserConfig]
-
-class BatchStatus(TypedDict, total=False):
-    """Status information for batch crawl operations."""
-    total: int
-    processed: int
-    succeeded: int
-    failed: int
-    in_progress: int
-    duration: float
-    
-class Pipeline:
-    """
-    A pipeline processor that executes a series of async middleware functions.
-    Each middleware function receives a context dictionary, updates it,
-    and returns 1 for success or 0 for failure.
-    """
-
-    def __init__(
-        self, 
-        middleware: List[Callable[[Dict[str, Any]], Awaitable[int]]] = None,
-        error_handler: Optional[Callable[[Dict[str, Any]], Awaitable[Dict[str, Any]]]] = None,
-        after_middleware_callback: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None,
-        crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
-        browser_config: Optional[BrowserConfig] = None,
-        logger: Optional[AsyncLogger] = None,
-        _initial_context: Optional[Dict[str, Any]] = None
-    ):
-        self.middleware = middleware or create_default_middleware_list()
-        self.error_handler = error_handler or handle_error_middleware
-        self.after_middleware_callback = after_middleware_callback
-        self.browser_config = browser_config or BrowserConfig()
-        self.logger = logger or AsyncLogger(verbose=self.browser_config.verbose)
-        self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
-            browser_config=self.browser_config,
-            logger=self.logger
-        )
-        self._initial_context = _initial_context
-        self._strategy_initialized = False
-    
-    async def _initialize_strategy__(self):
-        """Initialize the crawler strategy if not already initialized"""
-        if not self.crawler_strategy:
-            self.crawler_strategy = AsyncPlaywrightCrawlerStrategy(
-                browser_config=self.browser_config,
-                logger=self.logger
-            )
-        
-        if not self._strategy_initialized:
-            await self.crawler_strategy.__aenter__()
-            self._strategy_initialized = True
-    
-    async def _initialize_strategy(self):
-        """Initialize the crawler strategy if not already initialized"""
-        # With our new approach, we don't need to create the crawler strategy here
-        # as it will be created on-demand in fetch_content_middleware
-        
-        # Just ensure browser hub is available if needed
-        if hasattr(self, "_initial_context") and "browser_hub" not in self._initial_context:
-            # If a browser_config was provided but no browser_hub yet, 
-            # we'll let the browser_hub_middleware handle creating it
-            pass
-        
-        # Mark as initialized to prevent repeated initialization attempts
-        self._strategy_initialized = True
-
-    async def start(self):
-        """Start the crawler strategy and prepare it for use"""
-        if not self._strategy_initialized:
-            await self._initialize_strategy()
-            self._strategy_initialized = True
-        if self.crawler_strategy:
-            await self.crawler_strategy.__aenter__()
-            self._strategy_initialized = True
-        else:
-            raise ValueError("Crawler strategy is not initialized.")
-    
-    async def close(self):
-        """Close the crawler strategy and clean up resources"""
-        await self.stop()
-
-    async def stop(self):
-        """Close the crawler strategy and clean up resources"""
-        if self._strategy_initialized and self.crawler_strategy:
-            await self.crawler_strategy.__aexit__(None, None, None)
-            self._strategy_initialized = False
-    
-    async def __aenter__(self):
-        await self.start()
-        return self
-    
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        await self.close()
-    
-    async def crawl(self, url: str, config: Optional[CrawlerRunConfig] = None, **kwargs) -> CrawlResultContainer:
-        """
-        Crawl a URL and process it through the pipeline.
-        
-        Args:
-            url: The URL to crawl
-            config: Optional configuration for the crawl
-            **kwargs: Additional arguments to pass to the middleware
-            
-        Returns:
-            CrawlResultContainer: The result of the crawl
-        """
-        # Initialize strategy if needed
-        await self._initialize_strategy()
-        
-        # Create the initial context
-        context = {
-            "url": url,
-            "config": config or CrawlerRunConfig(),
-            "browser_config": self.browser_config,
-            "logger": self.logger,
-            "crawler_strategy": self.crawler_strategy,
-            "kwargs": kwargs
-        }
-        
-        # Process the pipeline
-        result_context = await self.process(context)
-        
-        # Return the final result
-        return result_context.get("final_result")
-    
-    async def process(self, initial_context: Dict[str, Any] = None) -> Dict[str, Any]:
-        """
-        Process all middleware functions with the given context.
-        
-        Args:
-            initial_context: Initial context dictionary, defaults to empty dict
-            
-        Returns:
-            Updated context dictionary after all middleware have been processed
-        """
-        context = {**self._initial_context}
-        if initial_context:
-            context.update(initial_context)
-        
-        # Record pipeline start time
-        context["_pipeline_start_time"] = time.perf_counter()
-        
-        for middleware_fn in self.middleware:
-            # Get middleware name for logging
-            middleware_name = getattr(middleware_fn, '__name__', str(middleware_fn))
-            
-            # Record start time for this middleware
-            start_time = time.perf_counter()
-            context[f"_timing_start_{middleware_name}"] = start_time
-            
-            try:
-                # Execute middleware (all middleware functions are async)
-                result = await middleware_fn(context)
-                
-                # Record completion time
-                end_time = time.perf_counter()
-                context[f"_timing_end_{middleware_name}"] = end_time
-                context[f"_timing_duration_{middleware_name}"] = end_time - start_time
-                
-                # Execute after-middleware callback if provided
-                if self.after_middleware_callback:
-                    await self.after_middleware_callback(middleware_name, context)
-                
-                # Convert boolean returns to int (True->1, False->0)
-                if isinstance(result, bool):
-                    result = 1 if result else 0
-                
-                # Handle failure
-                if result == 0:
-                    if self.error_handler:
-                        context["_error_in"] = middleware_name
-                        context["_error_at"] = time.perf_counter()
-                        return await self._handle_error(context)
-                    else:
-                        context["success"] = False
-                        context["error_message"] = f"Pipeline failed at {middleware_name}"
-                        break
-            except Exception as e:
-                # Record error information
-                context["_error_in"] = middleware_name
-                context["_error_at"] = time.perf_counter()
-                context["_exception"] = e
-                context["success"] = False
-                context["error_message"] = f"Exception in {middleware_name}: {str(e)}"
-                
-                # Call error handler if available
-                if self.error_handler:
-                    return await self._handle_error(context)
-                break
-        
-        # Record pipeline completion time
-        pipeline_end_time = time.perf_counter()
-        context["_pipeline_end_time"] = pipeline_end_time
-        context["_pipeline_duration"] = pipeline_end_time - context["_pipeline_start_time"]
-        
-        # Set success to True if not already set (no failures)
-        if "success" not in context:
-            context["success"] = True
-        
-        return context
-    
-    async def _handle_error(self, context: Dict[str, Any]) -> Dict[str, Any]:
-        """Handle errors by calling the error handler"""
-        try:
-            return await self.error_handler(context)
-        except Exception as e:
-            # If error handler fails, update context with this new error
-            context["_error_handler_exception"] = e
-            context["error_message"] = f"Error handler failed: {str(e)}"
-            return context
-
-
-
-async def create_pipeline(
-    middleware_list=None, 
-    error_handler=None,
-    after_middleware_callback=None,
-    browser_config=None,
-    browser_hub_id=None,
-    browser_hub_connection=None,
-    browser_hub=None,
-    logger=None
-) -> Pipeline:
-    """
-    Factory function to create a pipeline with Browser-Hub integration.
-    
-    Args:
-        middleware_list: List of middleware functions
-        error_handler: Error handler middleware
-        after_middleware_callback: Callback after middleware execution
-        browser_config: Configuration for the browser
-        browser_hub_id: ID for browser hub instance
-        browser_hub_connection: Connection string for existing browser hub
-        browser_hub: Existing browser hub instance to use
-        logger: Logger instance
-        
-    Returns:
-        Pipeline: Configured pipeline instance
-    """
-    # Use default middleware list if none provided
-    middleware = middleware_list or create_default_middleware_list()
-    
-    # Create the pipeline
-    pipeline = Pipeline(
-        middleware=middleware,
-        error_handler=error_handler,
-        after_middleware_callback=after_middleware_callback,
-        logger=logger
-    )
-    
-    # Set browser-related attributes in the initial context
-    pipeline._initial_context = {
-        "browser_config": browser_config,
-        "browser_hub_id": browser_hub_id,
-        "browser_hub_connection": browser_hub_connection,
-        "browser_hub": browser_hub,
-        "logger": logger
-    }
-    
-    return pipeline
-
-
-
-
-# async def create_pipeline(
-#     middleware_list: Optional[List[Callable[[Dict[str, Any]], Awaitable[int]]]] = None, 
-#     error_handler: Optional[Callable[[Dict[str, Any]], Awaitable[Dict[str, Any]]]] = None,
-#     after_middleware_callback: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None,
-#     crawler_strategy = None,
-#     browser_config = None,
-#     logger = None
-# ) -> Pipeline:
-#     """Factory function to create a pipeline with the given middleware"""
-#     return Pipeline(
-#         middleware=middleware_list,
-#         error_handler=error_handler,
-#         after_middleware_callback=after_middleware_callback,
-#         crawler_strategy=crawler_strategy,
-#         browser_config=browser_config,
-#         logger=logger
-#     )

crawl4ai/pipeline/test_pipeline.py

@@ -1,109 +0,0 @@
-import asyncio
-from crawl4ai import (
-    BrowserConfig,
-    CrawlerRunConfig,
-    CacheMode,
-    DefaultMarkdownGenerator,
-    PruningContentFilter
-)
-from pipeline import Pipeline
-
-async def main():
-    # Create configuration objects
-    browser_config = BrowserConfig(headless=True, verbose=True)
-    crawler_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(
-                threshold=0.48, 
-                threshold_type="fixed", 
-                min_word_threshold=0
-            )
-        ),
-    )
-    
-    # Create and use pipeline with context manager
-    async with Pipeline(browser_config=browser_config) as pipeline:
-        result = await pipeline.crawl(
-            url="https://www.example.com", 
-            config=crawler_config
-        )
-        
-        # Print the result
-        print(f"URL: {result.url}")
-        print(f"Success: {result.success}")
-        
-        if result.success:
-            print("\nMarkdown excerpt:")
-            print(result.markdown.raw_markdown[:500] + "...")
-        else:
-            print(f"Error: {result.error_message}")
-
-if __name__ == "__main__":
-    asyncio.run(main())
-
-
-class CrawlTarget:
-    def __init__(self, urls, config=None):
-        self.urls = urls
-        self.config = config
-
-    def __repr__(self):
-        return f"CrawlTarget(urls={self.urls}, config={self.config})"
-    
-
-
-
-# async def main():
-#     # Create configuration objects
-#     browser_config = BrowserConfig(headless=True, verbose=True)
-    
-#     # Define different configurations
-#     config1 = CrawlerRunConfig(
-#         cache_mode=CacheMode.BYPASS,
-#         markdown_generator=DefaultMarkdownGenerator(
-#             content_filter=PruningContentFilter(threshold=0.48)
-#         ),
-#     )
-    
-#     config2 = CrawlerRunConfig(
-#         cache_mode=CacheMode.ENABLED,
-#         screenshot=True,
-#         pdf=True
-#     )
-    
-#     # Create crawl targets
-#     targets = [
-#         CrawlTarget(
-#             urls=["https://www.example.com", "https://www.wikipedia.org"],
-#             config=config1
-#         ),
-#         CrawlTarget(
-#             urls="https://news.ycombinator.com",  
-#             config=config2
-#         ),
-#         CrawlTarget(
-#             urls=["https://github.com", "https://stackoverflow.com", "https://python.org"],
-#             config=None
-#         )
-#     ]
-    
-#     # Create and use pipeline with context manager
-#     async with Pipeline(browser_config=browser_config) as pipeline:
-#         all_results = await pipeline.crawl_batch(targets)
-        
-#         for target_key, results in all_results.items():
-#             print(f"\n===== Results for {target_key} =====")
-#             print(f"Number of URLs crawled: {len(results)}")
-            
-#             for i, result in enumerate(results):
-#                 print(f"\nURL {i+1}: {result.url}")
-#                 print(f"Success: {result.success}")
-                
-#                 if result.success:
-#                     print(f"Content length: {len(result.markdown.raw_markdown)} chars")
-#                 else:
-#                     print(f"Error: {result.error_message}")
-
-# if __name__ == "__main__":
-#     asyncio.run(main())

crawl4ai/processors/pdf/__init__.py

@@ -14,7 +14,7 @@ class PDFCrawlerStrategy(AsyncCrawlerStrategy):
     async def crawl(self, url: str, **kwargs) -> AsyncCrawlResponse:
         # Just pass through with empty HTML - scraper will handle actual processing
         return AsyncCrawlResponse(
-            html="",  # Scraper will handle the real work
+            html="Scraper will handle the real work",  # Scraper will handle the real work
             response_headers={"Content-Type": "application/pdf"},
             status_code=200
         )
@@ -66,6 +66,7 @@ class PDFContentScrapingStrategy(ContentScrapingStrategy):
             image_save_dir=image_save_dir,
             batch_size=batch_size
         )
+        self._temp_files = []  # Track temp files for cleanup
 
     def scrap(self, url: str, html: str, **params) -> ScrapingResult:
         """
@@ -124,7 +125,13 @@ class PDFContentScrapingStrategy(ContentScrapingStrategy):
         finally:
             # Cleanup temp file if downloaded
             if url.startswith(("http://", "https://")):
-                Path(pdf_path).unlink(missing_ok=True)
+                try:
+                    Path(pdf_path).unlink(missing_ok=True)
+                    if pdf_path in self._temp_files:
+                        self._temp_files.remove(pdf_path)
+                except Exception as e:
+                    if self.logger:
+                        self.logger.warning(f"Failed to cleanup temp file {pdf_path}: {e}")
 
     async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
         # For simple cases, you can use the sync version
@@ -138,22 +145,45 @@ class PDFContentScrapingStrategy(ContentScrapingStrategy):
             
             # Create temp file with .pdf extension
             temp_file = tempfile.NamedTemporaryFile(suffix='.pdf', delete=False)
+            self._temp_files.append(temp_file.name)
             
             try:
-                # Download PDF with streaming
-                response = requests.get(url, stream=True)
+                if self.logger:
+                    self.logger.info(f"Downloading PDF from {url}...")
+                
+                # Download PDF with streaming and timeout
+                # Connection timeout: 10s, Read timeout: 300s (5 minutes for large PDFs)
+                response = requests.get(url, stream=True, timeout=(20, 60 * 10))
                 response.raise_for_status()
                 
+                # Get file size if available
+                total_size = int(response.headers.get('content-length', 0))
+                downloaded = 0
+                
                 # Write to temp file
                 with open(temp_file.name, 'wb') as f:
                     for chunk in response.iter_content(chunk_size=8192):
                         f.write(chunk)
+                        downloaded += len(chunk)
+                        if self.logger and total_size > 0:
+                            progress = (downloaded / total_size) * 100
+                            if progress % 10 < 0.1:  # Log every 10%
+                                self.logger.debug(f"PDF download progress: {progress:.0f}%")
+                
+                if self.logger:
+                    self.logger.info(f"PDF downloaded successfully: {temp_file.name}")
                         
                 return temp_file.name
                 
+            except requests.exceptions.Timeout as e:
+                # Clean up temp file if download fails
+                Path(temp_file.name).unlink(missing_ok=True)
+                self._temp_files.remove(temp_file.name)
+                raise RuntimeError(f"Timeout downloading PDF from {url}: {str(e)}")
             except Exception as e:
                 # Clean up temp file if download fails
                 Path(temp_file.name).unlink(missing_ok=True)
+                self._temp_files.remove(temp_file.name)
                 raise RuntimeError(f"Failed to download PDF from {url}: {str(e)}")
                 
         elif url.startswith("file://"):

crawl4ai/prompts.py

@@ -1054,4 +1054,525 @@ Your output must:
 5. Include all required fields
 6. Use valid XPath selectors
 </output_requirements>
-"""
+"""
+
+GENERATE_SCRIPT_PROMPT = r"""You are a world-class browser automation specialist. Your sole purpose is to convert a natural language objective and a snippet of HTML into the most **efficient, robust, and simple** script possible to prepare a web page for data extraction.
+
+Your scripts run **before the crawl** to handle dynamic content, user interactions, and other obstacles. You are a master of two tools: raw **JavaScript** and the high-level **Crawl4ai Script (c4a)**.
+
+────────────────────────────────────────────────────────
+## Your Core Philosophy: "Efficiency, Robustness, Simplicity"
+
+This is your mantra. Every line of code you write must adhere to it.
+
+1.  **Efficiency (Shortest Path):** Generate the absolute minimum number of steps to achieve the goal. Do not include redundant actions. If a `CLICK` on one button achieves the goal, don't also scroll and wait unnecessarily.
+2.  **Robustness (Will Not Break):** Prioritize selectors and methods that are resistant to cosmetic site changes. `data-*` attributes are gold. Dynamic, auto-generated class names (`.class-a8B_x3`) are poison. Always prefer waiting for a state change (`WAIT \`#results\``) over a blind delay (`WAIT 5`).
+3.  **Simplicity (Right Tool for the Job):** Use the simplest tool that works. Prefer a direct `c4a` command over `EVAL` with JavaScript. Only use `EVAL` when the task is impossible with standard commands (e.g., accessing Shadow DOM, complex array filtering).
+
+────────────────────────────────────────────────────────
+## Output Mode Selection Logic
+
+Your choice of output mode is a critical strategic decision.
+
+*   **Use `crawl4ai_script` for:**
+    *   Standard, sequential browser actions: login forms, clicking "next page," simple "load more" buttons, accepting cookie banners.
+    *   When the user's goal maps clearly to the available `c4a` commands.
+    *   When you need to define reusable macros with `PROC`.
+
+*   **Use `javascript` for:**
+    *   Complex DOM manipulation that has no `c4a` equivalent (e.g., transforming data, complex filtering).
+    *   Interacting with web components inside **Shadow DOM** or **iFrames**.
+    *   Implementing sophisticated logic like custom scrolling patterns or handling non-standard events.
+    *   When the goal is a fine-grained DOM tweak, not a full user journey.
+
+**If the user specifies a mode, you MUST respect it.** If not, you must choose the mode that best embodies your core philosophy.
+
+────────────────────────────────────────────────────────
+## Available Crawl4ai Commands
+
+| Command                | Arguments / Notes                                            |
+|------------------------|--------------------------------------------------------------|
+| GO `<url>`             | Navigate to absolute URL                                     |
+| RELOAD                 | Hard refresh                                                |
+| BACK / FORWARD         | Browser history nav                                          |
+| WAIT `<seconds>`       | **Avoid!** Passive delay. Use only as a last resort.         |
+| WAIT \`<css>\` `<t>`   | **Preferred wait.** Poll selector until found, timeout in seconds. |
+| WAIT "<text>" `<t>`    | Poll page text until found, timeout in seconds.              |
+| CLICK \`<css>\`        | Single click on element                                     |
+| CLICK `<x>` `<y>`      | Viewport click                                              |
+| DOUBLE_CLICK …         | Two rapid clicks                                            |
+| RIGHT_CLICK …          | Context-menu click                                          |
+| MOVE `<x>` `<y>`       | Mouse move                                                  |
+| DRAG `<x1>` `<y1>` `<x2>` `<y2>` | Click-drag gesture                              |
+| SCROLL UP|DOWN|LEFT|RIGHT `[px]` | Viewport scroll                                 |
+| TYPE "<text>"          | Type into focused element                                   |
+| CLEAR \`<css>\`        | Empty input                                                 |
+| SET \`<css>\` "<val>"  | Set element value and dispatch events                       |
+| PRESS `<Key>`          | Keydown + keyup                                             |
+| KEY_DOWN `<Key>` / KEY_UP `<Key>` | Separate key events                          |
+| EVAL \`<js>\`          | **Your fallback.** Run JS when no direct command exists.     |
+| SETVAR $name = <val>   | Store constant for reuse                                    |
+| PROC name … ENDPROC    | Define macro                                                |
+| IF / ELSE / REPEAT     | Flow control                                                |
+| USE "<file.c4a>"       | Include another script, avoid circular includes             |
+
+────────────────────────────────────────────────────────
+## Strategic Principles & Anti-Patterns
+
+These are your commandments. Do not deviate.
+
+1.  **Selector Quality is Paramount:**
+    *   **GOOD:** `[data-testid="submit-button"]`, `#main-content`, `[aria-label="Close dialog"]`
+    *   **BAD:** `div > span:nth-child(3)`, `.button-gR3xY_s`, `//div[contains(@class, 'button')]`
+
+2.  **Wait for State, Not for Time:**
+    *   **DO:** `CLICK \`#load-more\`` followed by `WAIT \`div.new-item\` 10`. This waits for the *result* of the action.
+    *   **DON'T:** `CLICK \`#load-more\`` followed by `WAIT 5`. This is a guess and it will fail.
+
+3.  **Target the Action, Not the Artifact:** If you need to reveal content, click the button that reveals it. Don't try to manually change CSS `display` properties, as this can break the page's internal state.
+
+4.  **DOM-Awareness is Non-Negotiable:**
+    *   **Shadow DOM:** `c4a` commands CANNOT pierce the Shadow DOM. If you see a `#shadow-root (open)` in the HTML, you MUST use `EVAL` and `element.shadowRoot.querySelector(...)`.
+    *   **iFrames:** Likewise, you MUST use `EVAL` and `iframe.contentDocument.querySelector(...)` to interact with elements inside an iframe.
+
+5.  **Be Idempotent:** Your script must be harmless if run multiple times. Use `IF EXISTS` to check for states before acting (e.g., don't try to log in if already logged in).
+
+6.  **Forbidden Techniques:** Never use `document.write()`. It is destructive. Avoid overly complex JS in `EVAL` that could be simplified into a few `c4a` commands.
+
+────────────────────────────────────────────────────────
+## From Vague Goals to Robust Scripts: Your Duty to Infer and Ensure Reliability
+
+This is your most important responsibility. Users are not automation experts. They will provide incomplete or vague instructions. Your job is to be the expert—to infer their true goal and build a script that is reliable by default. You must add the "invisible scaffolding" of checks and waits to ensure the page is stable and ready for the crawler. **A vague user prompt must still result in a robust, complete script.**
+
+Study these examples. No matter which query is given, your output must be the single, robust solution.
+
+### 1. Scenario: Basic Search Query
+
+*   **High Detail Query:** "Find the search box and search button. Wait for the search box to be visible, click it, clear it, type 'r2d2', click the search button, and then wait for the search results to appear."
+*   **Medium Detail Query:** "Find the search box and search for 'r2d2', click the search button until you get a list of items."
+*   **Low Detail Query:** "Search for r2d2."
+
+**THE CORRECT, ROBUST OUTPUT (for all three queries):**
+```
+WAIT `input[type="search"]` 10
+SET `input[type="search"]` "r2d2"
+CLICK `button[aria-label="Search"]`
+WAIT `div.search-results-container` 15
+```
+**Rationale:** You correctly infer the need to `WAIT` for the input first. You use the more efficient `SET` command. Most importantly, you **infer the crucial final step**: waiting for a results container to appear, confirming the search action was successful.
+
+### 2. Scenario: Clicking a "Load More" Button
+
+*   **High Detail Query:** "Click the button with the text 'Load More'. Afterward, wait for a new item with the class '.product-tile' to show up on the page."
+*   **Medium Detail Query:** "Click the load more button to see more products."
+*   **Low Detail Query:** "Load more items."
+
+**THE CORRECT, ROBUST OUTPUT:**
+```
+IF EXISTS `button.load-more` THEN
+  CLICK `button.load-more`
+  WAIT `div.new-item-indicator` 8
+ENDIF
+```
+**Rationale:** You wrap the action in `IF EXISTS` to prevent errors if the button is not present (e.g., on the last page). You correctly infer the need to wait for the *consequence* of the click—a new item appearing—rather than a blind `WAIT 2`.
+
+### 3. Scenario: Applying a Filter from a Dropdown
+
+*   **High Detail Query:** "First, click the dropdown with id 'color-filter'. Then, from the open menu, click the option that says 'Blue'. Finally, wait for the product grid to update."
+*   **Medium Detail Query:** "Filter the products by the color Blue."
+*   **Low Detail Query:** "Show blue products."
+
+**THE CORRECT, ROBUST OUTPUT:**
+```
+CLICK `#color-filter`
+WAIT `[data-value="blue"]` 3
+CLICK `[data-value="blue"]`
+WAIT `#product-grid[data-status="updated"]` 10
+```
+**Rationale:** You infer the need for two waits: one for the menu options to appear after the first click, and a second for the main content to update after the filter is applied. This prevents race conditions.
+
+### 4. Scenario: User Authentication (Login)
+
+*   **High Detail Query:** "Fill username with 'USER_EMAIL', fill password with 'USER_PASS', click login, and wait for the dashboard to appear."
+*   **Medium Detail Query:** "Log in as USER_EMAIL with password USER_PASS."
+*   **Low Detail Query:** "Log in."
+
+**THE CORRECT, ROBUST OUTPUT:**
+```
+IF EXISTS `[data-testid="logout-button"]` THEN
+  EVAL `console.log("Already logged in.")`
+ELSE
+  WAIT `input[name="username"]` 10
+  SET `input[name="username"]` "USER_EMAIL"
+  SET `input[name="password"]` "USER_PASS"
+  CLICK `button[type="submit"]`
+  WAIT `[data-testid="user-dashboard"]` 15
+ENDIF
+```
+**Rationale:** You build an **idempotent** script. You first check if the user is *already* logged in. If not, you proceed with the login and then, critically, `WAIT` for a post-login element to confirm success. You use placeholders when credentials are not provided in low-detail queries.
+
+### 5. Scenario: Dismissing an Interstitial Modal
+
+*   **High Detail Query:** "Check if a popup with id '#promo-modal' exists. If it does, click the close button inside it with class '.close-x'."
+*   **Medium Detail Query:** "Close the promotional popup."
+*   **Low Detail Query:** "Get rid of the popup."
+
+**THE CORRECT, ROBUST OUTPUT:**
+```
+IF EXISTS `div#promo-modal` THEN
+  CLICK `div#promo-modal button.close-x`
+ENDIF
+```
+**Rationale:** You correctly identify this as a conditional action. The script must not fail if the popup doesn't appear. The `IF EXISTS` block is the perfect, robust way to handle this optional interaction.
+
+────────────────────────────────────────────────────────
+## Advanced Scenarios & Master-Level Examples
+
+Study these solutions. Understand the *why* behind each choice.
+
+### Scenario: Interacting with a Web Component (Shadow DOM)
+**Goal:** Click a button inside a custom element `<user-card>`.
+**HTML Snippet:** `<user-card><#shadow-root (open)><button>Details</button></#shadow-root></user-card>`
+**Correct Mode:** `javascript` (or `c4a` with `EVAL`)
+**Rationale:** Standard selectors can't cross the shadow boundary. JavaScript is mandatory.
+
+```javascript
+// Solution in pure JS mode
+const card = document.querySelector('user-card');
+if (card && card.shadowRoot) {
+  const button = card.shadowRoot.querySelector('button');
+  if (button) button.click();
+}
+```
+```
+# Solution in c4a mode (using EVAL as the weapon of choice)
+EVAL `
+  const card = document.querySelector('user-card');
+  if (card && card.shadowRoot) {
+    const button = card.shadowRoot.querySelector('button');
+    if (button) button.click();
+  }
+`
+```
+
+### Scenario: Handling a Cookie Banner
+**Goal:** Accept the cookies to dismiss the modal.
+**HTML Snippet:** `<div id="cookie-consent-modal"><button id="accept-cookies">Accept All</button></div>`
+**Correct Mode:** `crawl4ai_script`
+**Rationale:** A simple, direct action. `c4a` is cleaner and more declarative.
+
+```
+# The most efficient solution
+IF EXISTS `#cookie-consent-modal` THEN
+  CLICK `#accept-cookies`
+  WAIT `div.content-loaded` 5
+ENDIF
+```
+
+### Scenario: Infinite Scroll Page
+**Goal:** Scroll down 5 times to load more content.
+**HTML Snippet:** `(A page with a long body and no "load more" button)`
+**Correct Mode:** `crawl4ai_script`
+**Rationale:** `REPEAT` is designed for exactly this. It's more readable than a JS loop for this simple task.
+
+```
+REPEAT (
+  SCROLL DOWN 1000,
+  5
+)
+WAIT 2
+```
+
+### Scenario: Hover-to-Reveal Menu
+**Goal:** Hover over "Products" to open the menu, then click "Laptops".
+**HTML Snippet:** `<a href="/products" id="products-menu">Products</a> <div class="menu-dropdown"><a href="/laptops">Laptops</a></div>`
+**Correct Mode:** `crawl4ai_script` (with `EVAL`)
+**Rationale:** `c4a` has no `HOVER` command. `EVAL` is the perfect tool to dispatch the `mouseover` event.
+
+```
+EVAL `document.querySelector('#products-menu').dispatchEvent(new MouseEvent('mouseover', { bubbles: true }))`
+WAIT `div.menu-dropdown a[href="/laptops"]` 3
+CLICK `div.menu-dropdown a[href="/laptops"]`
+```
+
+### Scenario: Login Form
+**Goal:** Fill and submit a login form.
+**HTML Snippet:** `<form><input name="email"><input name="password" type="password"><button type="submit"></button></form>`
+**Correct Mode:** `crawl4ai_script`
+**Rationale:** This is the canonical use case for `c4a`. The commands map 1:1 to the user journey.
+
+```
+WAIT `form` 10
+SET `input[name="email"]` "USER_EMAIL"
+SET `input[name="password"]` "USER_PASS"
+CLICK `button[type="submit"]`
+WAIT `[data-testid="user-dashboard"]` 12
+```
+
+────────────────────────────────────────────────────────
+## Final Output Mandate
+
+1.  **CODE ONLY.** Your entire response must be the script body.
+2.  **NO CHAT.** Do not say "Here is the script" or "This should work."
+3.  **NO MARKDOWN.** Do not wrap your code in ` ``` ` fences.
+4.  **NO COMMENTS.** Do not add comments to the final code output.
+5.  **SYNTACTICALLY PERFECT.** The script must be immediately executable.
+6.  **UTF-8, STANDARD QUOTES.** Use `"` for string literals, not `“` or `”`.
+
+You are an engine of automation. Now, receive the user's request and produce the optimal script."""
+
+
+GENERATE_JS_SCRIPT_PROMPT = """# The World-Class JavaScript Automation Scripter
+
+You are a world-class browser automation specialist. Your sole purpose is to convert a natural language objective and a snippet of HTML into the most **efficient, robust, and simple** pure JavaScript script possible to prepare a web page for data extraction.
+
+Your scripts will be executed directly in the browser (e.g., via Playwright's `page.evaluate()`) to handle dynamic content, user interactions, and other obstacles before the page is crawled. You are a master of browser-native JavaScript APIs.
+
+────────────────────────────────────────────────────────
+## Your Core Philosophy: "Efficiency, Robustness, Simplicity"
+
+This is your mantra. Every line of JavaScript you write must adhere to it.
+
+1.  **Efficiency (Shortest Path):** Generate the absolute minimum number of steps to achieve the goal. Do not include redundant actions. Your code should be concise and direct.
+2.  **Robustness (Will Not Break):** Prioritize selectors that are resistant to cosmetic site changes. `data-*` attributes are gold. Dynamic, auto-generated class names (`.class-a8B_x3`) are poison. Always prefer waiting for a state change over a blind `setTimeout`.
+3.  **Simplicity (Right Tool for the Job):** Use simple, direct DOM methods (`.querySelector`, `.click()`) whenever possible. Avoid overly complex or fragile logic when a simpler approach exists.
+
+────────────────────────────────────────────────────────
+## Essential JavaScript Automation Patterns & Toolkit
+
+All code should be wrapped in an `async` Immediately Invoked Function Expression `(async () => { ... })();` to allow for top-level `await` and to avoid polluting the global scope.
+
+| Task                 | Best-Practice JavaScript Implementation                                                                                                                                                                                                                                                                                                      |
+| -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Wait for Element** | Create and use a robust `waitForElement` helper function. This is your most important tool. <br> `const waitForElement = (selector, timeout = 10000) => new Promise((resolve, reject) => { const el = document.querySelector(selector); if (el) return resolve(el); const observer = new MutationObserver(() => { const el = document.querySelector(selector); if (el) { observer.disconnect(); resolve(el); } }); observer.observe(document.body, { childList: true, subtree: true }); setTimeout(() => { observer.disconnect(); reject(new Error(`Timeout waiting for ${selector}`)); }, timeout); });` |
+| **Click Element**    | `const el = await waitForElement('selector'); if (el) el.click();`                                                                                                                                                                                                                                                                           |
+| **Set Input Value**  | `const input = await waitForElement('selector'); if (input) { input.value = 'new value'; input.dispatchEvent(new Event('input', { bubbles: true })); input.dispatchEvent(new Event('change', { bubbles: true })); }` <br> *Crucially, always dispatch `input` and `change` events to trigger framework reactivity.* |
+| **Check Existence**  | `const el = document.querySelector('selector'); if (el) { /* ... it exists */ }`                                                                                                                                                                                                                                                            |
+| **Scroll**           | `window.scrollBy(0, window.innerHeight);`                                                                                                                                                                                                                                                                                                    |
+| **Deal with Time**   | Use `await new Promise(r => setTimeout(r, 500));` for short, unavoidable pauses after an action. **Avoid long, blind waits.**                                                                                                                                                                                                                |
+
+REMEMBER: Make sure to generate very deterministic css selector. If you refer to a specific button, then be specific, otherwise you may capture elements you do not need, be very specific about the element you want to interact with.
+
+────────────────────────────────────────────────────────
+## The Art of High-Specificity Selectors: Your Defense Against Ambiguity
+
+This is your most critical skill for ensuring robustness. **You must assume the provided HTML is only a small fragment of the entire page.** A selector that looks unique in the fragment could be disastrously generic on the full page. Your primary defense is to **anchor your selectors to the most specific, stable parent element available in the given HTML context.**
+
+Think of it as creating a "sandbox" for your selectors.
+
+**Your Guiding Principle:** Start from a unique parent, then find the child.
+
+### Scenario: Selecting a Submit Button within a Login Form
+
+**HTML Snippet Provided:**
+```html
+<div class="user-auth-module" id="login-widget">
+  <h2>Member Login</h2>
+  <form action="/login">
+    <input name="email" type="email">
+    <input name="password" type="password">
+    <button type="submit">Sign In</button>
+  </form>
+</div>
+```
+
+*   **TERRIBLE (High Risk):** `button[type="submit"]`
+    *   **Why it's bad:** There could be dozens of other forms on the full page (e.g., a newsletter signup, a search bar in the header). This selector is a shot in the dark.
+
+*   **BETTER (Lower Risk):** `#login-widget button[type="submit"]`
+    *   **Why it's better:** It's anchored to a unique ID (`#login-widget`). This dramatically reduces the chance of ambiguity.
+
+*   **EXCELLENT (Minimal Risk):** `div[id="login-widget"] form button[type="submit"]`
+    *   **Why it's best:** This is a highly specific, descriptive path. It says, "Find the login widget, then the form inside it, and then the submit button inside *that* form." It is virtually guaranteed to be unique and is resilient to minor layout changes within the form.
+
+### Scenario: Selecting a "Add to Cart" Button
+
+**HTML Snippet Provided:**
+```html
+<section data-testid="product-details-main">
+  <h1>Awesome T-Shirt</h1>
+  <div class="product-actions">
+    <button class="add-to-cart-btn">Add to Cart</button>
+  </div>
+</section>
+```
+
+*   **TERRIBLE (High Risk):** `.add-to-cart-btn`
+    *   **Why it's bad:** A "related products" section outside this snippet might also use the same class name.
+
+*   **EXCELLENT (Minimal Risk):** `[data-testid="product-details-main"] .add-to-cart-btn`
+    *   **Why it's best:** It uses the stable `data-testid` attribute of the parent section as an anchor. This is the most robust pattern.
+
+**Your Mandate:** Always examine the provided HTML for a stable, unique parent (like an element with an `id`, a `data-testid`, or a highly specific combination of classes) and use it as the root of your selectors. **NEVER generate a generic, un-anchored selector if a better, more specific parent is available in the context.**
+
+
+────────────────────────────────────────────────────────
+## Strategic Principles & Anti-Patterns
+
+These are your commandments. Do not deviate.
+
+1.  **Selector Quality is Paramount:**
+    *   **GOOD:** `[data-testid="submit-button"]`, `#main-content`, `[aria-label="Close dialog"]`
+    *   **BAD:** `div > span:nth-child(3)`, `.button-gR3xY_s`, `//div[contains(@class, 'button')]`
+
+2.  **Wait for State, Not for Time:**
+    *   **DO:** `(await waitForElement('#load-more')).click(); await waitForElement('div.new-item');` This waits for the *result* of the action.
+    *   **DON'T:** `document.querySelector('#load-more').click(); await new Promise(r => setTimeout(r, 5000));` This is a guess and it will fail.
+
+3.  **Target the Action, Not the Artifact:** If you need to reveal content, click the button that reveals it. Don't try to manually change CSS `display` properties, as this can break the page's internal state.
+
+4.  **DOM-Awareness is Non-Negotiable:**
+    *   **Shadow DOM:** You MUST use `element.shadowRoot.querySelector(...)` to access elements inside a `#shadow-root (open)`.
+    *   **iFrames:** You MUST use `iframe.contentDocument.querySelector(...)` to interact with elements inside an iframe.
+
+5.  **Be Idempotent:** Your script must be harmless if run multiple times. Use `if (document.querySelector(...))` checks to avoid re-doing actions unnecessarily.
+
+6.  **Forbidden Techniques:** Never use `document.write()`. It is destructive.
+
+────────────────────────────────────────────────────────
+## From Vague Goals to Robust Scripts: Your Duty to Infer and Ensure Reliability
+
+This is your most important responsibility. Users are not automation experts. They will provide incomplete or vague instructions. Your job is to be the expert—to infer their true goal and build a script that is reliable by default. **A vague user prompt must still result in a robust, complete script.**
+
+Study these examples. No matter which query is given, your output must be the single, robust solution.
+
+### 1. Scenario: Basic Search Query
+
+*   **High Detail Query:** "Find the search box and search button. Wait for the search box to be visible, click it, clear it, type 'r2d2', click the search button, and then wait for the search results to appear."
+*   **Medium Detail Query:** "Find the search box and search for 'r2d2'."
+*   **Low Detail Query:** "Search for r2d2."
+
+**THE CORRECT, ROBUST JAVASCRIPT OUTPUT (for all three queries):**
+```javascript
+(async () => {
+  const waitForElement = (selector, timeout = 10000) => new Promise((resolve, reject) => { const el = document.querySelector(selector); if (el) return resolve(el); const observer = new MutationObserver(() => { const el = document.querySelector(selector); if (el) { observer.disconnect(); resolve(el); } }); observer.observe(document.body, { childList: true, subtree: true }); setTimeout(() => { observer.disconnect(); reject(new Error(`Timeout waiting for ${selector}`)); }, timeout); });
+  try {
+    const searchInput = await waitForElement('input[type="search"], input[aria-label*="search"]');
+    searchInput.value = 'r2d2';
+    searchInput.dispatchEvent(new Event('input', { bubbles: true }));
+    const searchButton = await waitForElement('button[type="submit"], button[aria-label*="search"]');
+    searchButton.click();
+    await waitForElement('div.search-results-container, #search-results');
+  } catch (e) {
+    console.error('Search script failed:', e.message);
+  }
+})();
+```
+
+### 2. Scenario: Clicking a "Load More" Button
+
+*   **High Detail Query:** "Click the button with the text 'Load More'. Afterward, wait for a new item with the class '.product-tile' to show up."
+*   **Medium Detail Query:** "Click the load more button."
+*   **Low Detail Query:** "Load more items."
+
+**THE CORRECT, ROBUST JAVASCRIPT OUTPUT:**
+```javascript
+(async () => {
+  const loadMoreButton = document.querySelector('button.load-more, [data-testid="load-more"]');
+  if (loadMoreButton) {
+    const initialItemCount = document.querySelectorAll('.product-tile').length;
+    loadMoreButton.click();
+    const waitForNewItem = (timeout = 8000) => new Promise((resolve, reject) => { const t0 = Date.now(); const check = () => { if (document.querySelectorAll('.product-tile').length > initialItemCount) return resolve(); if (Date.now() - t0 > timeout) return reject(new Error('Timeout waiting for new items to load.')); setTimeout(check, 200); }; check(); });
+    await waitForNewItem();
+  }
+})();
+```
+
+### 3. Scenario: User Authentication (Login)
+
+*   **High Detail Query:** "Fill username with 'USER_EMAIL', password with 'USER_PASS', click login, and wait for the dashboard."
+*   **Medium Detail Query:** "Log in as USER_EMAIL."
+*   **Low Detail Query:** "Log in."
+
+**THE CORRECT, ROBUST JAVASCRIPT OUTPUT:**
+```javascript
+(async () => {
+  if (document.querySelector('[data-testid="logout-button"]')) {
+    console.log('Already logged in.');
+    return;
+  }
+  const waitForElement = (selector, timeout = 10000) => new Promise((resolve, reject) => { const el = document.querySelector(selector); if (el) return resolve(el); const observer = new MutationObserver(() => { const el = document.querySelector(selector); if (el) { observer.disconnect(); resolve(el); } }); observer.observe(document.body, { childList: true, subtree: true }); setTimeout(() => { observer.disconnect(); reject(new Error(`Timeout waiting for ${selector}`)); }, timeout); });
+  try {
+    const userInput = await waitForElement('input[name*="user"], input[name*="email"]');
+    userInput.value = 'USER_EMAIL';
+    userInput.dispatchEvent(new Event('input', { bubbles: true }));
+    const passInput = await waitForElement('input[name*="pass"], input[type="password"]');
+    passInput.value = 'USER_PASS';
+    passInput.dispatchEvent(new Event('input', { bubbles: true }));
+    const submitButton = await waitForElement('button[type="submit"]');
+    submitButton.click();
+    await waitForElement('[data-testid="user-dashboard"], #dashboard, .account-page');
+  } catch (e) {
+    console.error('Login script failed:', e.message);
+  }
+})();
+```
+
+────────────────────────────────────────────────────────
+## The Art of High-Specificity Selectors: Your Defense Against Ambiguity
+
+This is your most critical skill for ensuring robustness. **You must assume the provided HTML is only a small fragment of the entire page.** A selector that looks unique in the fragment could be disastrously generic on the full page. Your primary defense is to **anchor your selectors to the most specific, stable parent element available in the given HTML context.**
+
+Think of it as creating a "sandbox" for your selectors.
+
+**Your Guiding Principle:** Start from a unique parent, then find the child.
+
+### Scenario: Selecting a Submit Button within a Login Form
+
+**HTML Snippet Provided:**
+```html
+<div class="user-auth-module" id="login-widget">
+  <h2>Member Login</h2>
+  <form action="/login">
+    <input name="email" type="email">
+    <input name="password" type="password">
+    <button type="submit">Sign In</button>
+  </form>
+</div>
+```
+
+*   **TERRIBLE (High Risk):** `button[type="submit"]`
+    *   **Why it's bad:** There could be dozens of other forms on the full page (e.g., a newsletter signup, a search bar in the header). This selector is a shot in the dark.
+
+*   **BETTER (Lower Risk):** `#login-widget button[type="submit"]`
+    *   **Why it's better:** It's anchored to a unique ID (`#login-widget`). This dramatically reduces the chance of ambiguity.
+
+*   **EXCELLENT (Minimal Risk):** `div[id="login-widget"] form button[type="submit"]`
+    *   **Why it's best:** This is a highly specific, descriptive path. It says, "Find the login widget, then the form inside it, and then the submit button inside *that* form." It is virtually guaranteed to be unique and is resilient to minor layout changes within the form.
+
+### Scenario: Selecting a "Add to Cart" Button
+
+**HTML Snippet Provided:**
+```html
+<section data-testid="product-details-main">
+  <h1>Awesome T-Shirt</h1>
+  <div class="product-actions">
+    <button class="add-to-cart-btn">Add to Cart</button>
+  </div>
+</section>
+```
+
+*   **TERRIBLE (High Risk):** `.add-to-cart-btn`
+    *   **Why it's bad:** A "related products" section outside this snippet might also use the same class name.
+
+*   **EXCELLENT (Minimal Risk):** `[data-testid="product-details-main"] .add-to-cart-btn`
+    *   **Why it's best:** It uses the stable `data-testid` attribute of the parent section as an anchor. This is the most robust pattern.
+
+**Your Mandate:** Always examine the provided HTML for a stable, unique parent (like an element with an `id`, a `data-testid`, or a highly specific combination of classes) and use it as the root of your selectors. **NEVER generate a generic, un-anchored selector if a better, more specific parent is available in the context.**
+
+
+────────────────────────────────────────────────────────
+## Final Output Mandate
+
+1.  **CODE ONLY.** Your entire response must be the script body.
+2.  **NO CHAT.** Do not say "Here is the script" or "This should work."
+3.  **NO MARKDOWN.** Do not wrap your code in ` ``` ` fences.
+4.  **NO COMMENTS.** Do not add comments to the final code output, except within the logic where it's a best practice.
+5.  **SYNTACTICALLY PERFECT.** The script must be a single, self-contained block, immediately executable. Wrap it in `(async () => { ... })();`.
+6.  **UTF-8, STANDARD QUOTES.** Use `'` for string literals, not `“` or `”`.
+
+You are an engine of automation. Now, receive the user's request and produce the optimal JavaScript."""
+
+
+
+

crawl4ai/proxy_strategy.py

@@ -4,6 +4,9 @@ from itertools import cycle
 import os
 
 
+########### ATTENTION PEOPLE OF EARTH ###########
+# I have moved this config to async_configs.py, kept it here, in case someone still importing it, however
+# be a dear and follow `from crawl4ai import ProxyConfig` instead :)
 class ProxyConfig:
     def __init__(
         self,
@@ -119,12 +122,12 @@ class ProxyRotationStrategy(ABC):
     """Base abstract class for proxy rotation strategies"""
     
     @abstractmethod
-    async def get_next_proxy(self) -> Optional[Dict]:
+    async def get_next_proxy(self) -> Optional[ProxyConfig]:
         """Get next proxy configuration from the strategy"""
         pass
 
     @abstractmethod
-    def add_proxies(self, proxies: List[Dict]):
+    def add_proxies(self, proxies: List[ProxyConfig]):
         """Add proxy configurations to the strategy"""
         pass
 

crawl4ai/script/__init__.py

@@ -0,0 +1,35 @@
+"""
+C4A-Script: A domain-specific language for web automation in Crawl4AI
+"""
+
+from .c4a_compile import C4ACompiler, compile, validate, compile_file
+from .c4a_result import (
+    CompilationResult, 
+    ValidationResult, 
+    ErrorDetail, 
+    WarningDetail,
+    ErrorType, 
+    Severity, 
+    Suggestion
+)
+
+__all__ = [
+    # Main compiler
+    "C4ACompiler",
+    
+    # Convenience functions
+    "compile",
+    "validate", 
+    "compile_file",
+    
+    # Result types
+    "CompilationResult",
+    "ValidationResult",
+    "ErrorDetail",
+    "WarningDetail",
+    
+    # Enums
+    "ErrorType",
+    "Severity",
+    "Suggestion"
+]

crawl4ai/script/c4a_compile.py

@@ -0,0 +1,398 @@
+"""
+Clean C4A-Script API with Result pattern
+No exceptions - always returns results
+"""
+
+from __future__ import annotations
+import pathlib
+import re
+from typing import Union, List, Optional
+
+# JSON_SCHEMA_BUILDER is still used elsewhere,
+# but we now also need the new script-builder prompt.
+from ..prompts import GENERATE_JS_SCRIPT_PROMPT, GENERATE_SCRIPT_PROMPT
+import logging
+import re
+
+from .c4a_result import (
+    CompilationResult, ValidationResult, ErrorDetail, WarningDetail,
+    ErrorType, Severity, Suggestion
+)
+from .c4ai_script import Compiler
+from lark.exceptions import UnexpectedToken, UnexpectedCharacters, VisitError
+from ..async_configs import LLMConfig
+from ..utils import perform_completion_with_backoff
+
+
+class C4ACompiler:
+    """Main compiler with result-based API"""
+    
+    # Error code mapping
+    ERROR_CODES = {
+        "missing_then": "E001",
+        "missing_paren": "E002",
+        "missing_comma": "E003",
+        "missing_endproc": "E004",
+        "undefined_proc": "E005",
+        "missing_backticks": "E006",
+        "invalid_command": "E007",
+        "syntax_error": "E999"
+    }
+    
+    @classmethod
+    def compile(cls, script: Union[str, List[str]], root: Optional[pathlib.Path] = None) -> CompilationResult:
+        """
+        Compile C4A-Script to JavaScript
+        
+        Args:
+            script: C4A-Script as string or list of lines
+            root: Root directory for includes
+            
+        Returns:
+            CompilationResult with success status and JS code or errors
+        """
+        # Normalize input
+        if isinstance(script, list):
+            script_text = '\n'.join(script)
+            script_lines = script
+        else:
+            script_text = script
+            script_lines = script.split('\n')
+        
+        try:
+            # Try compilation
+            compiler = Compiler(root)
+            js_code = compiler.compile(script_text)
+            
+            # Success!
+            result = CompilationResult(
+                success=True,
+                js_code=js_code,
+                metadata={
+                    "lineCount": len(script_lines),
+                    "statementCount": len(js_code)
+                }
+            )
+            
+            # Add any warnings (future feature)
+            # result.warnings = cls._check_warnings(script_text)
+            
+            return result
+            
+        except Exception as e:
+            # Convert exception to ErrorDetail
+            error = cls._exception_to_error(e, script_lines)
+            return CompilationResult(
+                success=False,
+                errors=[error],
+                metadata={
+                    "lineCount": len(script_lines)
+                }
+            )
+    
+    @classmethod
+    def validate(cls, script: Union[str, List[str]]) -> ValidationResult:
+        """
+        Validate script syntax without generating code
+        
+        Args:
+            script: C4A-Script to validate
+            
+        Returns:
+            ValidationResult with validity status and any errors
+        """
+        result = cls.compile(script)
+        
+        return ValidationResult(
+            valid=result.success,
+            errors=result.errors,
+            warnings=result.warnings
+        )
+    
+    @classmethod
+    def compile_file(cls, path: Union[str, pathlib.Path]) -> CompilationResult:
+        """
+        Compile a C4A-Script file
+        
+        Args:
+            path: Path to the file
+            
+        Returns:
+            CompilationResult
+        """
+        path = pathlib.Path(path)
+        
+        if not path.exists():
+            error = ErrorDetail(
+                type=ErrorType.RUNTIME,
+                code="E100",
+                severity=Severity.ERROR,
+                message=f"File not found: {path}",
+                line=0,
+                column=0,
+                source_line=""
+            )
+            return CompilationResult(success=False, errors=[error])
+        
+        try:
+            script = path.read_text()
+            return cls.compile(script, root=path.parent)
+        except Exception as e:
+            error = ErrorDetail(
+                type=ErrorType.RUNTIME,
+                code="E101",
+                severity=Severity.ERROR,
+                message=f"Error reading file: {str(e)}",
+                line=0,
+                column=0,
+                source_line=""
+            )
+            return CompilationResult(success=False, errors=[error])
+    
+    @classmethod
+    def _exception_to_error(cls, exc: Exception, script_lines: List[str]) -> ErrorDetail:
+        """Convert an exception to ErrorDetail"""
+        
+        if isinstance(exc, UnexpectedToken):
+            return cls._handle_unexpected_token(exc, script_lines)
+        elif isinstance(exc, UnexpectedCharacters):
+            return cls._handle_unexpected_chars(exc, script_lines)
+        elif isinstance(exc, ValueError):
+            return cls._handle_value_error(exc, script_lines)
+        else:
+            # Generic error
+            return ErrorDetail(
+                type=ErrorType.SYNTAX,
+                code=cls.ERROR_CODES["syntax_error"],
+                severity=Severity.ERROR,
+                message=str(exc),
+                line=1,
+                column=1,
+                source_line=script_lines[0] if script_lines else ""
+            )
+    
+    @classmethod
+    def _handle_unexpected_token(cls, exc: UnexpectedToken, script_lines: List[str]) -> ErrorDetail:
+        """Handle UnexpectedToken errors"""
+        line = exc.line
+        column = exc.column
+        
+        # Get context lines
+        source_line = script_lines[line - 1] if 0 < line <= len(script_lines) else ""
+        line_before = script_lines[line - 2] if line > 1 and line <= len(script_lines) + 1 else None
+        line_after = script_lines[line] if 0 < line < len(script_lines) else None
+        
+        # Determine error type and suggestions
+        if exc.token.type == 'CLICK' and 'THEN' in str(exc.expected):
+            code = cls.ERROR_CODES["missing_then"]
+            message = "Missing 'THEN' keyword after IF condition"
+            suggestions = [
+                Suggestion(
+                    "Add 'THEN' after the condition",
+                    source_line.replace("CLICK", "THEN CLICK") if source_line else None
+                )
+            ]
+        elif exc.token.type == '$END':
+            code = cls.ERROR_CODES["missing_endproc"]
+            message = "Unexpected end of script"
+            suggestions = [
+                Suggestion("Check for missing ENDPROC"),
+                Suggestion("Ensure all procedures are properly closed")
+            ]
+        elif 'RPAR' in str(exc.expected):
+            code = cls.ERROR_CODES["missing_paren"]
+            message = "Missing closing parenthesis ')'"
+            suggestions = [
+                Suggestion("Add closing parenthesis at the end of the condition")
+            ]
+        elif 'COMMA' in str(exc.expected):
+            code = cls.ERROR_CODES["missing_comma"]
+            message = "Missing comma ',' in command"
+            suggestions = [
+                Suggestion("Add comma between arguments")
+            ]
+        else:
+            # Check if this might be missing backticks
+            if exc.token.type == 'NAME' and 'BACKTICK_STRING' in str(exc.expected):
+                code = cls.ERROR_CODES["missing_backticks"]
+                message = "Selector must be wrapped in backticks"
+                suggestions = [
+                    Suggestion(
+                        "Wrap the selector in backticks",
+                        f"`{exc.token.value}`"
+                    )
+                ]
+            else:
+                code = cls.ERROR_CODES["syntax_error"]
+                message = f"Unexpected '{exc.token.value}'"
+                if exc.expected:
+                    expected_list = [str(e) for e in exc.expected if not str(e).startswith('_')][:3]
+                    if expected_list:
+                        message += f". Expected: {', '.join(expected_list)}"
+                suggestions = []
+        
+        return ErrorDetail(
+            type=ErrorType.SYNTAX,
+            code=code,
+            severity=Severity.ERROR,
+            message=message,
+            line=line,
+            column=column,
+            source_line=source_line,
+            line_before=line_before,
+            line_after=line_after,
+            suggestions=suggestions
+        )
+    
+    @classmethod
+    def _handle_unexpected_chars(cls, exc: UnexpectedCharacters, script_lines: List[str]) -> ErrorDetail:
+        """Handle UnexpectedCharacters errors"""
+        line = exc.line
+        column = exc.column
+        
+        source_line = script_lines[line - 1] if 0 < line <= len(script_lines) else ""
+        
+        # Check for missing backticks
+        if "CLICK" in source_line and column > source_line.find("CLICK"):
+            code = cls.ERROR_CODES["missing_backticks"]
+            message = "Selector must be wrapped in backticks"
+            suggestions = [
+                Suggestion(
+                    "Wrap the selector in backticks",
+                    re.sub(r'CLICK\s+([^\s]+)', r'CLICK `\1`', source_line)
+                )
+            ]
+        else:
+            code = cls.ERROR_CODES["syntax_error"]
+            message = f"Invalid character at position {column}"
+            suggestions = []
+        
+        return ErrorDetail(
+            type=ErrorType.SYNTAX,
+            code=code,
+            severity=Severity.ERROR,
+            message=message,
+            line=line,
+            column=column,
+            source_line=source_line,
+            suggestions=suggestions
+        )
+    
+    @classmethod
+    def _handle_value_error(cls, exc: ValueError, script_lines: List[str]) -> ErrorDetail:
+        """Handle ValueError (runtime errors)"""
+        message = str(exc)
+        
+        # Check for undefined procedure
+        if "Unknown procedure" in message:
+            proc_match = re.search(r"'([^']+)'", message)
+            if proc_match:
+                proc_name = proc_match.group(1)
+                
+                # Find the line with the procedure call
+                for i, line in enumerate(script_lines):
+                    if proc_name in line and not line.strip().startswith('PROC'):
+                        return ErrorDetail(
+                            type=ErrorType.RUNTIME,
+                            code=cls.ERROR_CODES["undefined_proc"],
+                            severity=Severity.ERROR,
+                            message=f"Undefined procedure '{proc_name}'",
+                            line=i + 1,
+                            column=line.find(proc_name) + 1,
+                            source_line=line,
+                            suggestions=[
+                                Suggestion(
+                                    f"Define the procedure before using it",
+                                    f"PROC {proc_name}\n  # commands here\nENDPROC"
+                                )
+                            ]
+                        )
+        
+        # Generic runtime error
+        return ErrorDetail(
+            type=ErrorType.RUNTIME,
+            code="E999",
+            severity=Severity.ERROR,
+            message=message,
+            line=1,
+            column=1,
+            source_line=script_lines[0] if script_lines else ""
+        )
+
+    @staticmethod
+    def generate_script(
+        html: str,
+        query: str | None = None,
+        mode: str = "c4a",
+        llm_config: LLMConfig | None = None,
+        **completion_kwargs,
+    ) -> str:
+        """
+        One-shot helper that calls the LLM exactly once to convert a
+        natural-language goal + HTML snippet into either:
+
+        1. raw JavaScript (`mode="js"`)
+        2. Crawl4ai DSL (`mode="c4a"`)
+
+        The returned string is guaranteed to be free of markdown wrappers
+        or explanatory text, ready for direct execution.
+        """
+        if llm_config is None:
+            llm_config = LLMConfig()  # falls back to env vars / defaults
+
+        # Build the user chunk
+        user_prompt = "\n".join(
+            [
+                "## GOAL",
+                "<<goael>>",
+                (query or "Prepare the page for crawling."),
+                "<</goal>>",
+                "",
+                "## HTML",
+                "<<html>>",
+                html[:100000],  # guardrail against token blast
+                "<</html>>",
+                "",
+                "## MODE",
+                mode,
+            ]
+        )
+
+        # Call the LLM with retry/back-off logic
+        full_prompt =  f"{GENERATE_SCRIPT_PROMPT}\n\n{user_prompt}" if mode == "c4a" else f"{GENERATE_JS_SCRIPT_PROMPT}\n\n{user_prompt}"
+        
+        response = perform_completion_with_backoff(
+            provider=llm_config.provider,
+            prompt_with_variables=full_prompt,
+            api_token=llm_config.api_token,
+            json_response=False,
+            base_url=getattr(llm_config, 'base_url', None),
+            **completion_kwargs,
+        )
+        
+        # Extract content from the response
+        raw_response = response.choices[0].message.content.strip()
+
+        # Strip accidental markdown fences (```js … ```)
+        clean = re.sub(r"^```(?:[a-zA-Z0-9_-]+)?\s*|```$", "", raw_response, flags=re.MULTILINE).strip()
+
+        if not clean:
+            raise RuntimeError("LLM returned empty script.")
+
+        return clean
+
+
+# Convenience functions for direct use
+def compile(script: Union[str, List[str]], root: Optional[pathlib.Path] = None) -> CompilationResult:
+    """Compile C4A-Script to JavaScript"""
+    return C4ACompiler.compile(script, root)
+
+
+def validate(script: Union[str, List[str]]) -> ValidationResult:
+    """Validate C4A-Script syntax"""
+    return C4ACompiler.validate(script)
+
+
+def compile_file(path: Union[str, pathlib.Path]) -> CompilationResult:
+    """Compile C4A-Script file"""
+    return C4ACompiler.compile_file(path)

crawl4ai/script/c4a_result.py

@@ -0,0 +1,219 @@
+"""
+Result classes for C4A-Script compilation
+Clean API design with no exceptions
+"""
+
+from __future__ import annotations
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import List, Dict, Any, Optional
+import json
+
+
+class ErrorType(Enum):
+    SYNTAX = "syntax"
+    SEMANTIC = "semantic"
+    RUNTIME = "runtime"
+    
+
+class Severity(Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+
+
+@dataclass
+class Suggestion:
+    """A suggestion for fixing an error"""
+    message: str
+    fix: Optional[str] = None
+    
+    def to_dict(self) -> dict:
+        return {
+            "message": self.message,
+            "fix": self.fix
+        }
+
+
+@dataclass
+class ErrorDetail:
+    """Detailed information about a compilation error"""
+    # Core info
+    type: ErrorType
+    code: str  # E001, E002, etc.
+    severity: Severity
+    message: str
+    
+    # Location
+    line: int
+    column: int
+    
+    # Context
+    source_line: str
+    
+    # Optional fields with defaults
+    end_line: Optional[int] = None
+    end_column: Optional[int] = None
+    line_before: Optional[str] = None
+    line_after: Optional[str] = None
+    
+    # Help
+    suggestions: List[Suggestion] = field(default_factory=list)
+    documentation_url: Optional[str] = None
+    
+    def to_dict(self) -> dict:
+        """Convert to dictionary for JSON serialization"""
+        return {
+            "type": self.type.value,
+            "code": self.code,
+            "severity": self.severity.value,
+            "message": self.message,
+            "location": {
+                "line": self.line,
+                "column": self.column,
+                "endLine": self.end_line,
+                "endColumn": self.end_column
+            },
+            "context": {
+                "sourceLine": self.source_line,
+                "lineBefore": self.line_before,
+                "lineAfter": self.line_after,
+                "marker": {
+                    "start": self.column - 1,
+                    "length": (self.end_column - self.column) if self.end_column else 1
+                }
+            },
+            "suggestions": [s.to_dict() for s in self.suggestions],
+            "documentationUrl": self.documentation_url
+        }
+    
+    def to_json(self) -> str:
+        """Convert to JSON string"""
+        return json.dumps(self.to_dict(), indent=2)
+    
+    @property
+    def formatted_message(self) -> str:
+        """Returns the nice text format for terminals"""
+        lines = []
+        lines.append(f"\n{'='*60}")
+        lines.append(f"{self.type.value.title()} Error [{self.code}]")
+        lines.append(f"{'='*60}")
+        lines.append(f"Location: Line {self.line}, Column {self.column}")
+        lines.append(f"Error: {self.message}")
+        
+        if self.source_line:
+            marker = " " * (self.column - 1) + "^"
+            if self.end_column:
+                marker += "~" * (self.end_column - self.column - 1)
+            lines.append(f"\nCode:")
+            if self.line_before:
+                lines.append(f"  {self.line - 1: >3} | {self.line_before}")
+            lines.append(f"  {self.line: >3} | {self.source_line}")
+            lines.append(f"      | {marker}")
+            if self.line_after:
+                lines.append(f"  {self.line + 1: >3} | {self.line_after}")
+        
+        if self.suggestions:
+            lines.append("\nSuggestions:")
+            for i, suggestion in enumerate(self.suggestions, 1):
+                lines.append(f"  {i}. {suggestion.message}")
+                if suggestion.fix:
+                    lines.append(f"     Fix: {suggestion.fix}")
+        
+        lines.append("="*60)
+        return "\n".join(lines)
+    
+    @property
+    def simple_message(self) -> str:
+        """Returns just the error message without formatting"""
+        return f"Line {self.line}: {self.message}"
+
+
+@dataclass
+class WarningDetail:
+    """Information about a compilation warning"""
+    code: str
+    message: str
+    line: int
+    column: int
+    
+    def to_dict(self) -> dict:
+        return {
+            "code": self.code,
+            "message": self.message,
+            "line": self.line,
+            "column": self.column
+        }
+
+
+@dataclass
+class CompilationResult:
+    """Result of C4A-Script compilation"""
+    success: bool
+    js_code: Optional[List[str]] = None
+    errors: List[ErrorDetail] = field(default_factory=list)
+    warnings: List[WarningDetail] = field(default_factory=list)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    def to_dict(self) -> dict:
+        """Convert to dictionary for JSON serialization"""
+        return {
+            "success": self.success,
+            "jsCode": self.js_code,
+            "errors": [e.to_dict() for e in self.errors],
+            "warnings": [w.to_dict() for w in self.warnings],
+            "metadata": self.metadata
+        }
+    
+    def to_json(self) -> str:
+        """Convert to JSON string"""
+        return json.dumps(self.to_dict(), indent=2)
+    
+    @property
+    def has_errors(self) -> bool:
+        """Check if there are any errors"""
+        return len(self.errors) > 0
+    
+    @property
+    def has_warnings(self) -> bool:
+        """Check if there are any warnings"""
+        return len(self.warnings) > 0
+    
+    @property
+    def first_error(self) -> Optional[ErrorDetail]:
+        """Get the first error if any"""
+        return self.errors[0] if self.errors else None
+    
+    def __str__(self) -> str:
+        """String representation for debugging"""
+        if self.success:
+            msg = f"✓ Compilation successful"
+            if self.js_code:
+                msg += f" - {len(self.js_code)} statements generated"
+            if self.warnings:
+                msg += f" ({len(self.warnings)} warnings)"
+            return msg
+        else:
+            return f"✗ Compilation failed - {len(self.errors)} error(s)"
+
+
+@dataclass
+class ValidationResult:
+    """Result of script validation"""
+    valid: bool
+    errors: List[ErrorDetail] = field(default_factory=list)
+    warnings: List[WarningDetail] = field(default_factory=list)
+    
+    def to_dict(self) -> dict:
+        return {
+            "valid": self.valid,
+            "errors": [e.to_dict() for e in self.errors],
+            "warnings": [w.to_dict() for w in self.warnings]
+        }
+    
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), indent=2)
+    
+    @property
+    def first_error(self) -> Optional[ErrorDetail]:
+        return self.errors[0] if self.errors else None

crawl4ai/script/c4ai_script.py

@@ -0,0 +1,690 @@
+"""
+2025-06-03
+By Unclcode:
+C4A-Script Language Documentation    
+Feeds Crawl4AI via CrawlerRunConfig(js_code=[ ... ]) – no core modifications.
+"""
+
+from __future__ import annotations
+import pathlib, re, sys, textwrap
+from dataclasses import dataclass
+from typing import Any, Dict, List, Union
+
+from lark import Lark, Transformer, v_args
+from lark.exceptions import UnexpectedToken, UnexpectedCharacters, VisitError
+
+
+# --------------------------------------------------------------------------- #
+# Custom Error Classes
+# --------------------------------------------------------------------------- #
+class C4AScriptError(Exception):
+    """Custom error class for C4A-Script compilation errors"""
+    
+    def __init__(self, message: str, line: int = None, column: int = None, 
+                 error_type: str = "Syntax Error", details: str = None):
+        self.message = message
+        self.line = line
+        self.column = column
+        self.error_type = error_type
+        self.details = details
+        super().__init__(self._format_message())
+    
+    def _format_message(self) -> str:
+        """Format a clear error message"""
+        lines = [f"\n{'='*60}"]
+        lines.append(f"C4A-Script {self.error_type}")
+        lines.append(f"{'='*60}")
+        
+        if self.line:
+            lines.append(f"Location: Line {self.line}" + (f", Column {self.column}" if self.column else ""))
+        
+        lines.append(f"Error: {self.message}")
+        
+        if self.details:
+            lines.append(f"\nDetails: {self.details}")
+        
+        lines.append("="*60)
+        return "\n".join(lines)
+    
+    @classmethod
+    def from_exception(cls, exc: Exception, script: Union[str, List[str]]) -> 'C4AScriptError':
+        """Create C4AScriptError from another exception"""
+        script_text = script if isinstance(script, str) else '\n'.join(script)
+        script_lines = script_text.split('\n')
+        
+        if isinstance(exc, UnexpectedToken):
+            # Extract line and column from UnexpectedToken
+            line = exc.line
+            column = exc.column
+            
+            # Get the problematic line
+            if 0 < line <= len(script_lines):
+                problem_line = script_lines[line - 1]
+                marker = " " * (column - 1) + "^"
+                
+                details = f"\nCode:\n  {problem_line}\n  {marker}\n"
+                
+                # Improve error message based on context
+                if exc.token.type == 'CLICK' and 'THEN' in str(exc.expected):
+                    message = "Missing 'THEN' keyword after IF condition"
+                elif exc.token.type == '$END':
+                    message = "Unexpected end of script. Check for missing ENDPROC or incomplete commands"
+                elif 'RPAR' in str(exc.expected):
+                    message = "Missing closing parenthesis ')'"
+                elif 'COMMA' in str(exc.expected):
+                    message = "Missing comma ',' in command"
+                else:
+                    message = f"Unexpected '{exc.token}'"
+                    if exc.expected:
+                        expected_list = [str(e) for e in exc.expected if not e.startswith('_')]
+                        if expected_list:
+                            message += f". Expected: {', '.join(expected_list[:3])}"
+                
+                details += f"Token: {exc.token.type} ('{exc.token.value}')"
+            else:
+                message = str(exc)
+                details = None
+            
+            return cls(message, line, column, "Syntax Error", details)
+        
+        elif isinstance(exc, UnexpectedCharacters):
+            # Extract line and column
+            line = exc.line
+            column = exc.column
+            
+            if 0 < line <= len(script_lines):
+                problem_line = script_lines[line - 1]
+                marker = " " * (column - 1) + "^"
+                
+                details = f"\nCode:\n  {problem_line}\n  {marker}\n"
+                message = f"Invalid character or unexpected text at position {column}"
+            else:
+                message = str(exc)
+                details = None
+            
+            return cls(message, line, column, "Syntax Error", details)
+        
+        elif isinstance(exc, ValueError):
+            # Handle runtime errors like undefined procedures
+            message = str(exc)
+            
+            # Try to find which line caused the error
+            if "Unknown procedure" in message:
+                proc_name = re.search(r"'([^']+)'", message)
+                if proc_name:
+                    proc_name = proc_name.group(1)
+                    for i, line in enumerate(script_lines, 1):
+                        if proc_name in line and not line.strip().startswith('PROC'):
+                            details = f"\nCode:\n  {line.strip()}\n\nMake sure the procedure '{proc_name}' is defined with PROC...ENDPROC"
+                            return cls(f"Undefined procedure '{proc_name}'", i, None, "Runtime Error", details)
+            
+            return cls(message, None, None, "Runtime Error", None)
+        
+        else:
+            # Generic error
+            return cls(str(exc), None, None, "Compilation Error", None)
+
+
+# --------------------------------------------------------------------------- #
+# 1. Grammar
+# --------------------------------------------------------------------------- #
+GRAMMAR = r"""
+start           : line*
+?line           : command | proc_def | include | comment
+
+command         : wait | nav | click_cmd | double_click | right_click | move | drag | scroll
+                | type | clear | set_input | press | key_down | key_up
+                | eval_cmd | setvar | proc_call | if_cmd | repeat_cmd
+
+wait            : "WAIT" (ESCAPED_STRING|BACKTICK_STRING|NUMBER) NUMBER? -> wait_cmd
+nav             : "GO" URL                             -> go
+                | "RELOAD"                             -> reload
+                | "BACK"                               -> back
+                | "FORWARD"                            -> forward
+
+click_cmd       : "CLICK" (BACKTICK_STRING|NUMBER NUMBER) -> click
+double_click    : "DOUBLE_CLICK" (BACKTICK_STRING|NUMBER NUMBER) -> double_click  
+right_click     : "RIGHT_CLICK" (BACKTICK_STRING|NUMBER NUMBER) -> right_click
+
+move            : "MOVE" coords                        -> move
+drag            : "DRAG" coords coords                 -> drag
+scroll          : "SCROLL" DIR NUMBER?                 -> scroll
+
+type            : "TYPE" (ESCAPED_STRING | NAME)       -> type
+clear           : "CLEAR" BACKTICK_STRING              -> clear
+set_input       : "SET" BACKTICK_STRING (ESCAPED_STRING | BACKTICK_STRING | NAME) -> set_input
+press           : "PRESS" WORD                         -> press
+key_down        : "KEY_DOWN" WORD                      -> key_down
+key_up          : "KEY_UP" WORD                        -> key_up
+
+eval_cmd        : "EVAL" BACKTICK_STRING               -> eval_cmd
+setvar          : "SETVAR" NAME "=" value              -> setvar
+proc_call       : NAME                                 -> proc_call
+proc_def        : "PROC" NAME line* "ENDPROC"          -> proc_def
+include         : "USE" ESCAPED_STRING                 -> include
+comment         : /#.*/                                -> comment
+
+if_cmd          : "IF" "(" condition ")" "THEN" command ("ELSE" command)?  -> if_cmd
+repeat_cmd      : "REPEAT" "(" command "," repeat_count ")"  -> repeat_cmd
+
+condition       : not_cond | exists_cond | js_cond
+not_cond        : "NOT" condition                      -> not_cond
+exists_cond     : "EXISTS" BACKTICK_STRING             -> exists_cond
+js_cond         : BACKTICK_STRING                      -> js_cond
+
+repeat_count    : NUMBER | BACKTICK_STRING
+
+coords          : NUMBER NUMBER
+value           : ESCAPED_STRING | BACKTICK_STRING | NUMBER
+DIR             : /(UP|DOWN|LEFT|RIGHT)/i
+REST            : /[^\n]+/
+
+URL             : /(http|https):\/\/[^\s]+/
+NAME            : /\$?[A-Za-z_][A-Za-z0-9_]*/
+WORD            : /[A-Za-z0-9+]+/
+BACKTICK_STRING : /`[^`]*`/
+
+%import common.NUMBER
+%import common.ESCAPED_STRING
+%import common.WS_INLINE
+%import common.NEWLINE
+%ignore WS_INLINE
+%ignore NEWLINE
+"""
+
+# --------------------------------------------------------------------------- #
+# 2. IR dataclasses
+# --------------------------------------------------------------------------- #
+@dataclass
+class Cmd:
+    op: str
+    args: List[Any]
+
+@dataclass
+class Proc:
+    name: str
+    body: List[Cmd]
+
+# --------------------------------------------------------------------------- #
+# 3. AST → IR
+# --------------------------------------------------------------------------- #
+@v_args(inline=True)
+class ASTBuilder(Transformer):
+    # helpers
+    def _strip(self, s): 
+        if s.startswith('"') and s.endswith('"'):
+            return s[1:-1]
+        elif s.startswith('`') and s.endswith('`'):
+            return s[1:-1]
+        return s
+    def start(self,*i):  return list(i)
+    def line(self,i):    return i
+    def command(self,i): return i
+
+    # WAIT
+    def wait_cmd(self, rest, timeout=None):
+        rest_str = str(rest)
+        # Check if it's a number (including floats)
+        try:
+            num_val = float(rest_str)
+            payload = (num_val, "seconds")
+        except ValueError:
+            if rest_str.startswith('"') and rest_str.endswith('"'):
+                payload = (self._strip(rest_str), "text")
+            elif rest_str.startswith('`') and rest_str.endswith('`'):
+                payload = (self._strip(rest_str), "selector")
+            else:
+                payload = (rest_str, "selector")
+        return Cmd("WAIT", [payload, int(timeout) if timeout else None])
+
+    # NAV
+    def go(self,u):    return Cmd("GO",[str(u)])
+    def reload(self):  return Cmd("RELOAD",[])
+    def back(self):    return Cmd("BACK",[])
+    def forward(self): return Cmd("FORWARD",[])
+
+    # CLICK, DOUBLE_CLICK, RIGHT_CLICK
+    def click(self, *args):
+        return self._handle_click("CLICK", args)
+    
+    def double_click(self, *args):
+        return self._handle_click("DBLCLICK", args)
+    
+    def right_click(self, *args):
+        return self._handle_click("RIGHTCLICK", args)
+    
+    def _handle_click(self, op, args):
+        if len(args) == 1:
+            # Single argument - backtick string
+            target = self._strip(str(args[0]))
+            return Cmd(op, [("selector", target)])
+        else:
+            # Two arguments - coordinates
+            x, y = args
+            return Cmd(op, [("coords", int(x), int(y))])
+
+
+    # MOVE / DRAG / SCROLL
+    def coords(self,x,y):       return ("coords",int(x),int(y))
+    def move(self,c):           return Cmd("MOVE",[c])
+    def drag(self,c1,c2):       return Cmd("DRAG",[c1,c2])
+    def scroll(self,dir_tok,amt=None):
+        return Cmd("SCROLL",[dir_tok.upper(), int(amt) if amt else 500])
+
+    # KEYS
+    def type(self,tok):     return Cmd("TYPE",[self._strip(str(tok))])
+    def clear(self,sel):    return Cmd("CLEAR",[self._strip(str(sel))])
+    def set_input(self,sel,val): return Cmd("SET",[self._strip(str(sel)), self._strip(str(val))])
+    def press(self,w):      return Cmd("PRESS",[str(w)])
+    def key_down(self,w):   return Cmd("KEYDOWN",[str(w)])
+    def key_up(self,w):     return Cmd("KEYUP",[str(w)])
+
+    # FLOW
+    def eval_cmd(self,txt):     return Cmd("EVAL",[self._strip(str(txt))])
+    def setvar(self,n,v):      
+        # v might be a Token or a Tree, extract value properly
+        if hasattr(v, 'value'):
+            value = v.value
+        elif hasattr(v, 'children') and len(v.children) > 0:
+            value = v.children[0].value
+        else:
+            value = str(v)
+        return Cmd("SETVAR",[str(n), self._strip(value)])
+    def proc_call(self,n):      return Cmd("CALL",[str(n)])
+    def proc_def(self,n,*body): return Proc(str(n),[b for b in body if isinstance(b,Cmd)])
+    def include(self,p):        return Cmd("INCLUDE",[self._strip(p)])
+    def comment(self,*_):       return Cmd("NOP",[])
+    
+    # IF-THEN-ELSE and EXISTS
+    def if_cmd(self, condition, then_cmd, else_cmd=None):
+        return Cmd("IF", [condition, then_cmd, else_cmd])
+    
+    def condition(self, cond):
+        return cond
+    
+    def not_cond(self, cond):
+        return ("NOT", cond)
+    
+    def exists_cond(self, selector):
+        return ("EXISTS", self._strip(str(selector)))
+    
+    def js_cond(self, expr):
+        return ("JS", self._strip(str(expr)))
+    
+    # REPEAT
+    def repeat_cmd(self, cmd, count):
+        return Cmd("REPEAT", [cmd, count])
+    
+    def repeat_count(self, value):
+        return str(value)
+
+# --------------------------------------------------------------------------- #
+# 4. Compiler
+# --------------------------------------------------------------------------- #
+class Compiler:
+    def __init__(self, root: pathlib.Path|None=None):
+        self.parser = Lark(GRAMMAR,start="start",parser="lalr")
+        self.root   = pathlib.Path(root or ".").resolve()
+        self.vars: Dict[str,Any] = {}
+        self.procs: Dict[str,Proc]= {}
+
+    def compile(self, text: Union[str, List[str]]) -> List[str]:
+        # Handle list input by joining with newlines
+        if isinstance(text, list):
+            text = '\n'.join(text)
+        
+        ir = self._parse_with_includes(text)
+        ir = self._collect_procs(ir)
+        ir = self._inline_calls(ir)
+        ir = self._apply_set_vars(ir)
+        return [self._emit_js(c) for c in ir if isinstance(c,Cmd) and c.op!="NOP"]
+
+    # passes
+    def _parse_with_includes(self,txt,seen=None):
+        seen=seen or set()
+        cmds=ASTBuilder().transform(self.parser.parse(txt))
+        out=[]
+        for c in cmds:
+            if isinstance(c,Cmd) and c.op=="INCLUDE":
+                p=(self.root/c.args[0]).resolve()
+                if p in seen: raise ValueError(f"Circular include {p}")
+                seen.add(p); out+=self._parse_with_includes(p.read_text(),seen)
+            else: out.append(c)
+        return out
+
+    def _collect_procs(self,ir):
+        out=[]
+        for i in ir:
+            if isinstance(i,Proc): self.procs[i.name]=i
+            else: out.append(i)
+        return out
+
+    def _inline_calls(self,ir):
+        out=[]
+        for c in ir:
+            if isinstance(c,Cmd) and c.op=="CALL":
+                if c.args[0] not in self.procs:
+                    raise ValueError(f"Unknown procedure {c.args[0]!r}")
+                out+=self._inline_calls(self.procs[c.args[0]].body)
+            else: out.append(c)
+        return out
+
+    def _apply_set_vars(self,ir):
+        def sub(s): return re.sub(r"\$(\w+)",lambda m:str(self.vars.get(m.group(1),m.group(0))) ,s) if isinstance(s,str) else s
+        out=[]
+        for c in ir:
+            if isinstance(c,Cmd):
+                if c.op=="SETVAR":
+                    # Store variable
+                    self.vars[c.args[0].lstrip('$')]=c.args[1]
+                else:
+                    # Apply variable substitution to commands that use them
+                    if c.op in("TYPE","EVAL","SET"): c.args=[sub(a) for a in c.args]
+                    out.append(c)
+        return out
+
+    # JS emitter
+    def _emit_js(self, cmd: Cmd) -> str:
+        op, a = cmd.op, cmd.args
+        if op == "GO":         return f"window.location.href = '{a[0]}';"
+        if op == "RELOAD":     return "window.location.reload();"
+        if op == "BACK":       return "window.history.back();"
+        if op == "FORWARD":    return "window.history.forward();"
+
+        if op == "WAIT":
+            arg, kind = a[0]
+            timeout   = a[1] or 10
+            if kind == "seconds":
+                return f"await new Promise(r=>setTimeout(r,{arg}*1000));"
+            if kind == "selector":
+                sel = arg.replace("\\","\\\\").replace("'","\\'")
+                return textwrap.dedent(f"""
+                    await new Promise((res,rej)=>{{
+                      const max = {timeout*1000}, t0 = performance.now();
+                      const id = setInterval(()=>{{
+                        if(document.querySelector('{sel}')){{clearInterval(id);res();}}
+                        else if(performance.now()-t0>max){{clearInterval(id);rej('WAIT selector timeout');}}
+                      }},100);
+                    }});
+                """).strip()
+            if kind == "text":
+                txt = arg.replace('`', '\\`')
+                return textwrap.dedent(f"""
+                    await new Promise((res,rej)=>{{
+                      const max={timeout*1000},t0=performance.now();
+                      const id=setInterval(()=>{{
+                        if(document.body.innerText.includes(`{txt}`)){{clearInterval(id);res();}}
+                        else if(performance.now()-t0>max){{clearInterval(id);rej('WAIT text timeout');}}
+                      }},100);
+                    }});
+                """).strip()
+
+        # click-style helpers
+        def _js_click(sel, evt="click", button=0, detail=1):
+            sel = sel.replace("'", "\\'")
+            return textwrap.dedent(f"""
+                (()=>{{
+                  const el=document.querySelector('{sel}');
+                  if(el){{
+                    el.focus&&el.focus();
+                    el.dispatchEvent(new MouseEvent('{evt}',{{bubbles:true,button:{button},detail:{detail}}}));
+                  }}
+                }})();
+            """).strip()
+
+        def _js_click_xy(x, y, evt="click", button=0, detail=1):
+            return textwrap.dedent(f"""
+                (()=>{{
+                  const el=document.elementFromPoint({x},{y});
+                  if(el){{
+                    el.focus&&el.focus();
+                    el.dispatchEvent(new MouseEvent('{evt}',{{bubbles:true,button:{button},detail:{detail}}}));
+                  }}
+                }})();
+            """).strip()
+
+        if op in ("CLICK", "DBLCLICK", "RIGHTCLICK"):
+            evt   = {"CLICK":"click","DBLCLICK":"dblclick","RIGHTCLICK":"contextmenu"}[op]
+            btn   = 2 if op=="RIGHTCLICK" else 0
+            det   = 2 if op=="DBLCLICK"   else 1
+            kind,*rest = a[0]
+            return _js_click_xy(*rest) if kind=="coords" else _js_click(rest[0],evt,btn,det)
+
+        if op == "MOVE":
+            _, x, y = a[0]
+            return textwrap.dedent(f"""
+                document.dispatchEvent(new MouseEvent('mousemove',{{clientX:{x},clientY:{y},bubbles:true}}));
+            """).strip()
+
+        if op == "DRAG":
+            (_, x1, y1), (_, x2, y2) = a
+            return textwrap.dedent(f"""
+                (()=>{{
+                  const s=document.elementFromPoint({x1},{y1});
+                  if(!s) return;
+                  s.dispatchEvent(new MouseEvent('mousedown',{{bubbles:true,clientX:{x1},clientY:{y1}}}));
+                  document.dispatchEvent(new MouseEvent('mousemove',{{bubbles:true,clientX:{x2},clientY:{y2}}}));
+                  document.dispatchEvent(new MouseEvent('mouseup',  {{bubbles:true,clientX:{x2},clientY:{y2}}}));
+                }})();
+            """).strip()
+
+        if op == "SCROLL":
+            dir_, amt = a
+            dx, dy = {"UP":(0,-amt),"DOWN":(0,amt),"LEFT":(-amt,0),"RIGHT":(amt,0)}[dir_]
+            return f"window.scrollBy({dx},{dy});"
+
+        if op == "TYPE":
+            txt = a[0].replace("'", "\\'")
+            return textwrap.dedent(f"""
+                (()=>{{
+                  const el=document.activeElement;
+                  if(el){{
+                    el.value += '{txt}';
+                    el.dispatchEvent(new Event('input',{{bubbles:true}}));
+                  }}
+                }})();
+            """).strip()
+
+        if op == "CLEAR":
+            sel = a[0].replace("'", "\\'")
+            return textwrap.dedent(f"""
+                (()=>{{
+                  const el=document.querySelector('{sel}');
+                  if(el && 'value' in el){{
+                    el.value = '';
+                    el.dispatchEvent(new Event('input',{{bubbles:true}}));
+                    el.dispatchEvent(new Event('change',{{bubbles:true}}));
+                  }}
+                }})();
+            """).strip()
+
+        if op == "SET" and len(a) == 2:
+            # This is SET for input fields (SET `#field` "value")
+            sel = a[0].replace("'", "\\'")
+            val = a[1].replace("'", "\\'")
+            return textwrap.dedent(f"""
+                (()=>{{
+                  const el=document.querySelector('{sel}');
+                  if(el && 'value' in el){{
+                    el.value = '';
+                    el.focus&&el.focus();
+                    el.value = '{val}';
+                    el.dispatchEvent(new Event('input',{{bubbles:true}}));
+                    el.dispatchEvent(new Event('change',{{bubbles:true}}));
+                  }}
+                }})();
+            """).strip()
+
+        if op in ("PRESS","KEYDOWN","KEYUP"):
+            key = a[0]
+            evs = {"PRESS":("keydown","keyup"),"KEYDOWN":("keydown",),"KEYUP":("keyup",)}[op]
+            return ";".join([f"document.dispatchEvent(new KeyboardEvent('{e}',{{key:'{key}',bubbles:true}}))" for e in evs]) + ";"
+
+        if op == "EVAL":
+            return textwrap.dedent(f"""
+                (()=>{{
+                  try {{
+                    {a[0]};
+                  }} catch (e) {{
+                    console.error('C4A-Script EVAL error:', e);
+                  }}
+                }})();
+            """).strip()
+        
+        if op == "IF":
+            condition, then_cmd, else_cmd = a
+            
+            # Generate condition JavaScript
+            js_condition = self._emit_condition(condition)
+            
+            # Generate commands - handle both regular commands and procedure calls
+            then_js = self._handle_cmd_or_proc(then_cmd)
+            else_js = self._handle_cmd_or_proc(else_cmd) if else_cmd else ""
+            
+            if else_cmd:
+                return textwrap.dedent(f"""
+                    if ({js_condition}) {{
+                      {then_js}
+                    }} else {{
+                      {else_js}
+                    }}
+                """).strip()
+            else:
+                return textwrap.dedent(f"""
+                    if ({js_condition}) {{
+                      {then_js}
+                    }}
+                """).strip()
+        
+        if op == "REPEAT":
+            cmd, count = a
+            
+            # Handle the count - could be number or JS expression
+            if count.isdigit():
+                # Simple number
+                repeat_js = self._handle_cmd_or_proc(cmd)
+                return textwrap.dedent(f"""
+                    for (let _i = 0; _i < {count}; _i++) {{
+                      {repeat_js}
+                    }}
+                """).strip()
+            else:
+                # JS expression (from backticks)
+                count_expr = count[1:-1] if count.startswith('`') and count.endswith('`') else count
+                repeat_js = self._handle_cmd_or_proc(cmd)
+                return textwrap.dedent(f"""
+                    (()=>{{
+                      const _count = {count_expr};
+                      if (typeof _count === 'number') {{
+                        for (let _i = 0; _i < _count; _i++) {{
+                          {repeat_js}
+                        }}
+                      }} else if (_count) {{
+                        {repeat_js}
+                      }}
+                    }})();
+                """).strip()
+
+        raise ValueError(f"Unhandled op {op}")
+    
+    def _emit_condition(self, condition):
+        """Convert a condition tuple to JavaScript"""
+        cond_type = condition[0]
+        
+        if cond_type == "EXISTS":
+            return f"!!document.querySelector('{condition[1]}')"
+        elif cond_type == "NOT":
+            # Recursively handle the negated condition
+            inner_condition = self._emit_condition(condition[1])
+            return f"!({inner_condition})"
+        else:  # JS condition
+            return condition[1]
+    
+    def _handle_cmd_or_proc(self, cmd):
+        """Handle a command that might be a regular command or a procedure call"""
+        if not cmd:
+            return ""
+        
+        if isinstance(cmd, Cmd):
+            if cmd.op == "CALL":
+                # Inline the procedure
+                if cmd.args[0] not in self.procs:
+                    raise ValueError(f"Unknown procedure {cmd.args[0]!r}")
+                proc_body = self.procs[cmd.args[0]].body
+                return "\n".join([self._emit_js(c) for c in proc_body if c.op != "NOP"])
+            else:
+                return self._emit_js(cmd)
+        return ""
+
+# --------------------------------------------------------------------------- #
+# 5. Helpers + demo
+# --------------------------------------------------------------------------- #
+
+def compile_string(script: Union[str, List[str]], *, root: Union[pathlib.Path, None] = None) -> List[str]:
+    """Compile C4A-Script from string or list of strings to JavaScript.
+    
+    Args:
+        script: C4A-Script as a string or list of command strings
+        root: Root directory for resolving includes (optional)
+    
+    Returns:
+        List of JavaScript command strings
+    
+    Raises:
+        C4AScriptError: When compilation fails with detailed error information
+    """
+    try:
+        return Compiler(root).compile(script)
+    except Exception as e:
+        # Wrap the error with better formatting
+        raise C4AScriptError.from_exception(e, script)
+
+def compile_file(path: pathlib.Path) -> List[str]:
+    """Compile C4A-Script from file to JavaScript.
+    
+    Args:
+        path: Path to C4A-Script file
+    
+    Returns:
+        List of JavaScript command strings
+    """
+    return compile_string(path.read_text(), root=path.parent)
+
+def compile_lines(lines: List[str], *, root: Union[pathlib.Path, None] = None) -> List[str]:
+    """Compile C4A-Script from list of lines to JavaScript.
+    
+    Args:
+        lines: List of C4A-Script command lines
+        root: Root directory for resolving includes (optional)
+    
+    Returns:
+        List of JavaScript command strings
+    """
+    return compile_string(lines, root=root)
+
+DEMO = """
+# quick sanity demo
+PROC login
+  SET `input[name="username"]` $user
+  SET `input[name="password"]` $pass
+  CLICK `button.submit`
+ENDPROC
+
+SETVAR user = "tom@crawl4ai.com"
+SETVAR pass = "hunter2"
+
+GO https://example.com/login
+WAIT `input[name="username"]` 10
+login
+WAIT 3
+EVAL `console.log('logged in')`
+"""
+
+if __name__ == "__main__":
+    if len(sys.argv) == 2:
+        for js in compile_file(pathlib.Path(sys.argv[1])):
+            print(js)
+    else:
+        print("=== DEMO ===")
+        for js in compile_string(DEMO):
+            print(js)

crawl4ai/ssl_certificate.py

@@ -9,83 +9,44 @@ from urllib.parse import urlparse
 import OpenSSL.crypto
 from pathlib import Path
 
-
-class SSLCertificate:
+# === Inherit from dict ===
+class SSLCertificate(dict):
     """
-    A class representing an SSL certificate with methods to export in various formats.
+    A class representing an SSL certificate, behaving like a dictionary
+    for direct JSON serialization. It stores the certificate information internally
+    and provides methods for export and property access.
 
-    Attributes:
-        cert_info (Dict[str, Any]): The certificate information.
-
-        Methods:
-            from_url(url: str, timeout: int = 10) -> Optional['SSLCertificate']: Create SSLCertificate instance from a URL.
-            from_file(file_path: str) -> Optional['SSLCertificate']: Create SSLCertificate instance from a file.
-            from_binary(binary_data: bytes) -> Optional['SSLCertificate']: Create SSLCertificate instance from binary data.
-            export_as_pem() -> str: Export the certificate as PEM format.
-            export_as_der() -> bytes: Export the certificate as DER format.
-            export_as_json() -> Dict[str, Any]: Export the certificate as JSON format.
-            export_as_text() -> str: Export the certificate as text format.
+    Inherits from dict, so instances are directly JSON serializable.
     """
 
+    # Use __slots__ for potential memory optimization if desired, though less common when inheriting dict
+    # __slots__ = ("_cert_info",) # If using slots, be careful with dict inheritance interaction
+
     def __init__(self, cert_info: Dict[str, Any]):
-        self._cert_info = self._decode_cert_data(cert_info)
-
-    @staticmethod
-    def from_url(url: str, timeout: int = 10) -> Optional["SSLCertificate"]:
         """
-        Create SSLCertificate instance from a URL.
+        Initializes the SSLCertificate object.
 
         Args:
-            url (str): URL of the website.
-            timeout (int): Timeout for the connection (default: 10).
-
-        Returns:
-            Optional[SSLCertificate]: SSLCertificate instance if successful, None otherwise.
+            cert_info (Dict[str, Any]): The raw certificate dictionary.
         """
-        try:
-            hostname = urlparse(url).netloc
-            if ":" in hostname:
-                hostname = hostname.split(":")[0]
+        # 1. Decode the data (handle bytes -> str)
+        decoded_info = self._decode_cert_data(cert_info)
 
-            context = ssl.create_default_context()
-            with socket.create_connection((hostname, 443), timeout=timeout) as sock:
-                with context.wrap_socket(sock, server_hostname=hostname) as ssock:
-                    cert_binary = ssock.getpeercert(binary_form=True)
-                    x509 = OpenSSL.crypto.load_certificate(
-                        OpenSSL.crypto.FILETYPE_ASN1, cert_binary
-                    )
+        # 2. Store the decoded info internally (optional but good practice)
+        # self._cert_info = decoded_info # You can keep this if methods rely on it
 
-                    cert_info = {
-                        "subject": dict(x509.get_subject().get_components()),
-                        "issuer": dict(x509.get_issuer().get_components()),
-                        "version": x509.get_version(),
-                        "serial_number": hex(x509.get_serial_number()),
-                        "not_before": x509.get_notBefore(),
-                        "not_after": x509.get_notAfter(),
-                        "fingerprint": x509.digest("sha256").hex(),
-                        "signature_algorithm": x509.get_signature_algorithm(),
-                        "raw_cert": base64.b64encode(cert_binary),
-                    }
-
-                    # Add extensions
-                    extensions = []
-                    for i in range(x509.get_extension_count()):
-                        ext = x509.get_extension(i)
-                        extensions.append(
-                            {"name": ext.get_short_name(), "value": str(ext)}
-                        )
-                    cert_info["extensions"] = extensions
-
-                    return SSLCertificate(cert_info)
-
-        except Exception:
-            return None
+        # 3. Initialize the dictionary part of the object with the decoded data
+        super().__init__(decoded_info)
 
     @staticmethod
     def _decode_cert_data(data: Any) -> Any:
         """Helper method to decode bytes in certificate data."""
         if isinstance(data, bytes):
-            return data.decode("utf-8")
+            try:
+                # Try UTF-8 first, fallback to latin-1 for arbitrary bytes
+                return data.decode("utf-8")
+            except UnicodeDecodeError:
+                return data.decode("latin-1") # Or handle as needed, maybe hex representation
         elif isinstance(data, dict):
             return {
                 (
@@ -97,36 +58,119 @@ class SSLCertificate:
             return [SSLCertificate._decode_cert_data(item) for item in data]
         return data
 
+    @staticmethod
+    def from_url(url: str, timeout: int = 10) -> Optional["SSLCertificate"]:
+        """
+        Create SSLCertificate instance from a URL. Fetches cert info and initializes.
+        (Fetching logic remains the same)
+        """
+        cert_info_raw = None # Variable to hold the fetched dict
+        try:
+            hostname = urlparse(url).netloc
+            if ":" in hostname:
+                hostname = hostname.split(":")[0]
+
+            context = ssl.create_default_context()
+            # Set check_hostname to False and verify_mode to CERT_NONE temporarily
+            # for potentially problematic certificates during fetch, but parse the result regardless.
+            # context.check_hostname = False
+            # context.verify_mode = ssl.CERT_NONE
+
+            with socket.create_connection((hostname, 443), timeout=timeout) as sock:
+                with context.wrap_socket(sock, server_hostname=hostname) as ssock:
+                    cert_binary = ssock.getpeercert(binary_form=True)
+                    if not cert_binary:
+                         print(f"Warning: No certificate returned for {hostname}")
+                         return None
+
+                    x509 = OpenSSL.crypto.load_certificate(
+                        OpenSSL.crypto.FILETYPE_ASN1, cert_binary
+                    )
+
+                    # Create the dictionary directly
+                    cert_info_raw = {
+                        "subject": dict(x509.get_subject().get_components()),
+                        "issuer": dict(x509.get_issuer().get_components()),
+                        "version": x509.get_version(),
+                        "serial_number": hex(x509.get_serial_number()),
+                        "not_before": x509.get_notBefore(), # Keep as bytes initially, _decode handles it
+                        "not_after": x509.get_notAfter(),   # Keep as bytes initially
+                        "fingerprint": x509.digest("sha256").hex(), # hex() is already string
+                        "signature_algorithm": x509.get_signature_algorithm(), # Keep as bytes
+                        "raw_cert": base64.b64encode(cert_binary), # Base64 is bytes, _decode handles it
+                    }
+
+                    # Add extensions
+                    extensions = []
+                    for i in range(x509.get_extension_count()):
+                        ext = x509.get_extension(i)
+                        # get_short_name() returns bytes, str(ext) handles value conversion
+                        extensions.append(
+                            {"name": ext.get_short_name(), "value": str(ext)}
+                        )
+                    cert_info_raw["extensions"] = extensions
+
+        except ssl.SSLCertVerificationError as e:
+             print(f"SSL Verification Error for {url}: {e}")
+             # Decide if you want to proceed or return None based on your needs
+             # You might try fetching without verification here if needed, but be cautious.
+             return None
+        except socket.gaierror:
+            print(f"Could not resolve hostname: {hostname}")
+            return None
+        except socket.timeout:
+            print(f"Connection timed out for {url}")
+            return None
+        except Exception as e:
+            print(f"Error fetching/processing certificate for {url}: {e}")
+            # Log the full error details if needed: logging.exception("Cert fetch error")
+            return None
+
+        # If successful, create the SSLCertificate instance from the dictionary
+        if cert_info_raw:
+             return SSLCertificate(cert_info_raw)
+        else:
+             return None
+
+
+    # --- Properties now access the dictionary items directly via self[] ---
+    @property
+    def issuer(self) -> Dict[str, str]:
+        return self.get("issuer", {}) # Use self.get for safety
+
+    @property
+    def subject(self) -> Dict[str, str]:
+        return self.get("subject", {})
+
+    @property
+    def valid_from(self) -> str:
+        return self.get("not_before", "")
+
+    @property
+    def valid_until(self) -> str:
+        return self.get("not_after", "")
+
+    @property
+    def fingerprint(self) -> str:
+        return self.get("fingerprint", "")
+
+    # --- Export methods can use `self` directly as it is the dict ---
     def to_json(self, filepath: Optional[str] = None) -> Optional[str]:
-        """
-        Export certificate as JSON.
-
-        Args:
-            filepath (Optional[str]): Path to save the JSON file (default: None).
-
-        Returns:
-            Optional[str]: JSON string if successful, None otherwise.
-        """
-        json_str = json.dumps(self._cert_info, indent=2, ensure_ascii=False)
+        """Export certificate as JSON."""
+        # `self` is already the dictionary we want to serialize
+        json_str = json.dumps(self, indent=2, ensure_ascii=False)
         if filepath:
             Path(filepath).write_text(json_str, encoding="utf-8")
             return None
         return json_str
 
     def to_pem(self, filepath: Optional[str] = None) -> Optional[str]:
-        """
-        Export certificate as PEM.
-
-        Args:
-            filepath (Optional[str]): Path to save the PEM file (default: None).
-
-        Returns:
-            Optional[str]: PEM string if successful, None otherwise.
-        """
+        """Export certificate as PEM."""
         try:
+            # Decode the raw_cert (which should be string due to _decode)
+            raw_cert_bytes = base64.b64decode(self.get("raw_cert", ""))
             x509 = OpenSSL.crypto.load_certificate(
-                OpenSSL.crypto.FILETYPE_ASN1,
-                base64.b64decode(self._cert_info["raw_cert"]),
+                OpenSSL.crypto.FILETYPE_ASN1, raw_cert_bytes
             )
             pem_data = OpenSSL.crypto.dump_certificate(
                 OpenSSL.crypto.FILETYPE_PEM, x509
@@ -136,49 +180,25 @@ class SSLCertificate:
                 Path(filepath).write_text(pem_data, encoding="utf-8")
                 return None
             return pem_data
-        except Exception:
-            return None
+        except Exception as e:
+             print(f"Error converting to PEM: {e}")
+             return None
 
     def to_der(self, filepath: Optional[str] = None) -> Optional[bytes]:
-        """
-        Export certificate as DER.
-
-        Args:
-            filepath (Optional[str]): Path to save the DER file (default: None).
-
-        Returns:
-            Optional[bytes]: DER bytes if successful, None otherwise.
-        """
+        """Export certificate as DER."""
         try:
-            der_data = base64.b64decode(self._cert_info["raw_cert"])
+            # Decode the raw_cert (which should be string due to _decode)
+            der_data = base64.b64decode(self.get("raw_cert", ""))
             if filepath:
                 Path(filepath).write_bytes(der_data)
                 return None
             return der_data
-        except Exception:
-            return None
+        except Exception as e:
+             print(f"Error converting to DER: {e}")
+             return None
 
-    @property
-    def issuer(self) -> Dict[str, str]:
-        """Get certificate issuer information."""
-        return self._cert_info.get("issuer", {})
-
-    @property
-    def subject(self) -> Dict[str, str]:
-        """Get certificate subject information."""
-        return self._cert_info.get("subject", {})
-
-    @property
-    def valid_from(self) -> str:
-        """Get certificate validity start date."""
-        return self._cert_info.get("not_before", "")
-
-    @property
-    def valid_until(self) -> str:
-        """Get certificate validity end date."""
-        return self._cert_info.get("not_after", "")
-
-    @property
-    def fingerprint(self) -> str:
-        """Get certificate fingerprint."""
-        return self._cert_info.get("fingerprint", "")
+    # Optional: Add __repr__ for better debugging
+    def __repr__(self) -> str:
+        subject_cn = self.subject.get('CN', 'N/A')
+        issuer_cn = self.issuer.get('CN', 'N/A')
+        return f"<SSLCertificate Subject='{subject_cn}' Issuer='{issuer_cn}'>"

crawl4ai/types.py

@@ -10,17 +10,22 @@ CacheMode = Union['CacheModeType']
 CrawlResult = Union['CrawlResultType']
 CrawlerHub = Union['CrawlerHubType']
 BrowserProfiler = Union['BrowserProfilerType']
+# NEW: Add AsyncUrlSeederType
+AsyncUrlSeeder = Union['AsyncUrlSeederType']
 
 # Configuration types
 BrowserConfig = Union['BrowserConfigType']
 CrawlerRunConfig = Union['CrawlerRunConfigType']
 HTTPCrawlerConfig = Union['HTTPCrawlerConfigType']
 LLMConfig = Union['LLMConfigType']
+# NEW: Add SeedingConfigType
+SeedingConfig = Union['SeedingConfigType']
 
 # Content scraping types
 ContentScrapingStrategy = Union['ContentScrapingStrategyType']
-WebScrapingStrategy = Union['WebScrapingStrategyType']
 LXMLWebScrapingStrategy = Union['LXMLWebScrapingStrategyType']
+# Backward compatibility alias
+WebScrapingStrategy = Union['LXMLWebScrapingStrategyType']
 
 # Proxy types
 ProxyRotationStrategy = Union['ProxyRotationStrategyType']
@@ -94,6 +99,8 @@ if TYPE_CHECKING:
     from .models import CrawlResult as CrawlResultType
     from .hub import CrawlerHub as CrawlerHubType
     from .browser_profiler import BrowserProfiler as BrowserProfilerType
+    # NEW: Import AsyncUrlSeeder for type checking
+    from .async_url_seeder import AsyncUrlSeeder as AsyncUrlSeederType
     
     # Configuration imports
     from .async_configs import (
@@ -101,12 +108,13 @@ if TYPE_CHECKING:
         CrawlerRunConfig as CrawlerRunConfigType,
         HTTPCrawlerConfig as HTTPCrawlerConfigType,
         LLMConfig as LLMConfigType,
+        # NEW: Import SeedingConfig for type checking
+        SeedingConfig as SeedingConfigType,
     )
     
     # Content scraping imports
     from .content_scraping_strategy import (
         ContentScrapingStrategy as ContentScrapingStrategyType,
-        WebScrapingStrategy as WebScrapingStrategyType,
         LXMLWebScrapingStrategy as LXMLWebScrapingStrategyType,
     )
     
@@ -184,4 +192,4 @@ if TYPE_CHECKING:
 
 def create_llm_config(*args, **kwargs) -> 'LLMConfigType':
     from .async_configs import LLMConfig
-    return LLMConfig(*args, **kwargs)
+    return LLMConfig(*args, **kwargs)

crawl4ai/utils.py

@@ -6,6 +6,7 @@ import html
 import lxml
 import re
 import os
+import subprocess
 import platform
 from .prompts import PROMPT_EXTRACT_BLOCKS
 from array import array
@@ -20,7 +21,6 @@ from urllib.parse import urljoin
 import requests
 from requests.exceptions import InvalidSchema
 import xxhash
-from colorama import Fore, Style, init
 import textwrap
 import cProfile
 import pstats
@@ -32,7 +32,6 @@ import hashlib
 
 from urllib.robotparser import RobotFileParser
 import aiohttp
-from urllib.parse import urlparse, urlunparse
 from functools import lru_cache
 
 from packaging import version
@@ -43,6 +42,37 @@ from itertools import chain
 from collections import deque
 from typing import  Generator, Iterable
 
+import numpy as np
+
+from urllib.parse import (
+    urljoin, urlparse, urlunparse,
+    parse_qsl, urlencode, quote, unquote
+)
+
+
+# Monkey patch to fix wildcard handling in urllib.robotparser
+from urllib.robotparser import RuleLine
+import re
+
+original_applies_to = RuleLine.applies_to
+
+def patched_applies_to(self, filename):
+   # Handle wildcards in paths
+   if '*' in self.path or '%2A' in self.path or self.path in ("*", "%2A"):
+       pattern = self.path.replace('%2A', '*')
+       pattern = re.escape(pattern).replace('\\*', '.*')
+       pattern = '^' + pattern
+       if pattern.endswith('\\$'):
+           pattern = pattern[:-2] + '$'
+       try:
+           return bool(re.match(pattern, filename))
+       except re.error:
+           return original_applies_to(self, filename)
+   return original_applies_to(self, filename)
+
+RuleLine.applies_to = patched_applies_to
+# Monkey patch ends
+
 def chunk_documents(
     documents: Iterable[str],
     chunk_token_threshold: int,
@@ -136,13 +166,20 @@ def merge_chunks(
     word_token_ratio: float = 1.0,
     splitter: Callable = None
 ) -> List[str]:
-    """Merges documents into chunks of specified token size.
+    """
+    Merges a sequence of documents into chunks based on a target token count, with optional overlap.
+    
+    Each document is split into tokens using the provided splitter function (defaults to str.split). Tokens are distributed into chunks aiming for the specified target size, with optional overlapping tokens between consecutive chunks. Returns a list of non-empty merged chunks as strings.
     
     Args:
-        docs: Input documents
-        target_size: Desired token count per chunk
-        overlap: Number of tokens to overlap between chunks
-        word_token_ratio: Multiplier for word->token conversion
+        docs: Sequence of input document strings to be merged.
+        target_size: Target number of tokens per chunk.
+        overlap: Number of tokens to overlap between consecutive chunks.
+        word_token_ratio: Multiplier to estimate token count from word count.
+        splitter: Callable used to split each document into tokens.
+    
+    Returns:
+        List of merged document chunks as strings, each not exceeding the target token size.
     """
     # Pre-tokenize all docs and store token counts
     splitter = splitter or str.split
@@ -151,7 +188,7 @@ def merge_chunks(
     total_tokens = 0
     
     for doc in docs:
-        tokens = doc.split()
+        tokens = splitter(doc)
         count = int(len(tokens) * word_token_ratio)
         if count:  # Skip empty docs
             token_counts.append(count)
@@ -304,7 +341,7 @@ class RobotsParser:
                 robots_url = f"{scheme}://{domain}/robots.txt"
                 
                 async with aiohttp.ClientSession() as session:
-                    async with session.get(robots_url, timeout=2) as response:
+                    async with session.get(robots_url, timeout=2, ssl=False) as response:
                         if response.status == 200:
                             rules = await response.text()
                             self._cache_rules(domain, rules)
@@ -441,14 +478,13 @@ def create_box_message(
         str: A formatted string containing the styled message box.
     """
 
-    init()
-
     # Define border and text colors for different types
     styles = {
-        "warning": (Fore.YELLOW, Fore.LIGHTYELLOW_EX, "⚠"),
-        "info": (Fore.BLUE, Fore.LIGHTBLUE_EX, "ℹ"),
-        "success": (Fore.GREEN, Fore.LIGHTGREEN_EX, "✓"),
-        "error": (Fore.RED, Fore.LIGHTRED_EX, "×"),
+        "warning": ("yellow", "bright_yellow", "⚠"),
+        "info": ("blue", "bright_blue", "ℹ"),
+        "debug": ("lightblack", "bright_black", "⋯"),
+        "success": ("green", "bright_green", "✓"),
+        "error": ("red", "bright_red", "×"),
     }
 
     border_color, text_color, prefix = styles.get(type.lower(), styles["info"])
@@ -480,12 +516,12 @@ def create_box_message(
     # Create the box with colored borders and lighter text
     horizontal_line = h_line * (width - 1)
     box = [
-        f"{border_color}{tl}{horizontal_line}{tr}",
+        f"[{border_color}]{tl}{horizontal_line}{tr}[/{border_color}]",
         *[
-            f"{border_color}{v_line}{text_color} {line:<{width-2}}{border_color}{v_line}"
+            f"[{border_color}]{v_line}[{text_color}] {line:<{width-2}}[/{text_color}][{border_color}]{v_line}[/{border_color}]"
             for line in formatted_lines
         ],
-        f"{border_color}{bl}{horizontal_line}{br}{Style.RESET_ALL}",
+        f"[{border_color}]{bl}{horizontal_line}{br}[/{border_color}]",
     ]
 
     result = "\n".join(box)
@@ -1111,6 +1147,23 @@ def get_content_of_website_optimized(
     css_selector: str = None,
     **kwargs,
 ) -> Dict[str, Any]:
+    """
+    Extracts and cleans content from website HTML, optimizing for useful media and contextual information.
+    
+    Parses the provided HTML to extract internal and external links, filters and scores images for usefulness, gathers contextual descriptions for media, removes unwanted or low-value elements, and converts the cleaned HTML to Markdown. Also extracts metadata and returns all structured content in a dictionary.
+    
+    Args:
+        url: The URL of the website being processed.
+        html: The raw HTML content to extract from.
+        word_count_threshold: Minimum word count for elements to be retained.
+        css_selector: Optional CSS selector to restrict extraction to specific elements.
+    
+    Returns:
+        A dictionary containing Markdown content, cleaned HTML, extraction success status, media and link lists, and metadata.
+    
+    Raises:
+        InvalidCSSSelectorError: If a provided CSS selector does not match any elements.
+    """
     if not html:
         return None
 
@@ -1153,6 +1206,20 @@ def get_content_of_website_optimized(
 
     def process_image(img, url, index, total_images):
         # Check if an image has valid display and inside undesired html elements
+        """
+        Processes an HTML image element to determine its relevance and extract metadata.
+        
+        Evaluates an image's visibility, context, and usefulness based on its attributes and parent elements. If the image passes validation and exceeds a usefulness score threshold, returns a dictionary with its source, alt text, contextual description, score, and type. Otherwise, returns None.
+        
+        Args:
+            img: The BeautifulSoup image tag to process.
+            url: The base URL of the page containing the image.
+            index: The index of the image in the list of images on the page.
+            total_images: The total number of images on the page.
+        
+        Returns:
+            A dictionary with image metadata if the image is considered useful, or None otherwise.
+        """
         def is_valid_image(img, parent, parent_classes):
             style = img.get("style", "")
             src = img.get("src", "")
@@ -1174,6 +1241,20 @@ def get_content_of_website_optimized(
         # Score an image for it's usefulness
         def score_image_for_usefulness(img, base_url, index, images_count):
             # Function to parse image height/width value and units
+            """
+            Scores an HTML image element for usefulness based on size, format, attributes, and position.
+            
+            The function evaluates an image's dimensions, file format, alt text, and its position among all images on the page to assign a usefulness score. Higher scores indicate images that are likely more relevant or informative for content extraction or summarization.
+            
+            Args:
+                img: The HTML image element to score.
+                base_url: The base URL used to resolve relative image sources.
+                index: The position of the image in the list of images on the page (zero-based).
+                images_count: The total number of images on the page.
+            
+            Returns:
+                An integer usefulness score for the image.
+            """
             def parse_dimension(dimension):
                 if dimension:
                     match = re.match(r"(\d+)(\D*)", dimension)
@@ -1188,6 +1269,16 @@ def get_content_of_website_optimized(
             # Fetch image file metadata to extract size and extension
             def fetch_image_file_size(img, base_url):
                 # If src is relative path construct full URL, if not it may be CDN URL
+                """
+                Fetches the file size of an image by sending a HEAD request to its URL.
+                
+                Args:
+                    img: A BeautifulSoup tag representing the image element.
+                    base_url: The base URL to resolve relative image sources.
+                
+                Returns:
+                    The value of the "Content-Length" header as a string if available, otherwise None.
+                """
                 img_url = urljoin(base_url, img.get("src"))
                 try:
                     response = requests.head(img_url)
@@ -1198,8 +1289,6 @@ def get_content_of_website_optimized(
                         return None
                 except InvalidSchema:
                     return None
-                finally:
-                    return
 
             image_height = img.get("height")
             height_value, height_unit = parse_dimension(image_height)
@@ -1428,8 +1517,29 @@ def extract_metadata_using_lxml(html, doc=None):
     head = head[0]
 
     # Title - using XPath
+    # title = head.xpath(".//title/text()")
+    # metadata["title"] = title[0].strip() if title else None
+
+    # === Title Extraction - New Approach ===
+    # Attempt to extract <title> using XPath
     title = head.xpath(".//title/text()")
-    metadata["title"] = title[0].strip() if title else None
+    title = title[0] if title else None
+
+    # Fallback: Use .find() in case XPath fails due to malformed HTML
+    if not title:
+        title_el = doc.find(".//title")
+        title = title_el.text if title_el is not None else None
+
+    # Final fallback: Use OpenGraph or Twitter title if <title> is missing or empty
+    if not title:
+        title_candidates = (
+            doc.xpath("//meta[@property='og:title']/@content") or
+            doc.xpath("//meta[@name='twitter:title']/@content")
+        )
+        title = title_candidates[0] if title_candidates else None
+
+    # Strip and assign title
+    metadata["title"] = title.strip() if title else None
 
     # Meta description - using XPath with multiple attribute conditions
     description = head.xpath('.//meta[@name="description"]/@content')
@@ -1458,6 +1568,14 @@ def extract_metadata_using_lxml(html, doc=None):
         content = tag.get("content", "").strip()
         if property_name and content:
             metadata[property_name] = content
+   
+   # Article metadata
+    article_tags = head.xpath('.//meta[starts-with(@property, "article:")]')
+    for tag in article_tags:
+        property_name = tag.get("property", "").strip()
+        content = tag.get("content", "").strip()
+        if property_name and content:
+            metadata[property_name] = content
 
     return metadata
 
@@ -1533,7 +1651,15 @@ def extract_metadata(html, soup=None):
         content = tag.get("content", "").strip()
         if property_name and content:
             metadata[property_name] = content
-
+    
+    # Article metadata
+    article_tags = head.find_all("meta", attrs={"property": re.compile(r"^article:")})
+    for tag in article_tags:
+        property_name = tag.get("property", "").strip()
+        content = tag.get("content", "").strip()
+        if property_name and content:
+            metadata[property_name] = content
+    
     return metadata
 
 
@@ -2002,9 +2128,101 @@ def normalize_url(href, base_url):
     parsed_base = urlparse(base_url)
     if not parsed_base.scheme or not parsed_base.netloc:
         raise ValueError(f"Invalid base URL format: {base_url}")
+    
+    if  parsed_base.scheme.lower() not in ["http", "https"]:
+        # Handle special protocols
+        raise ValueError(f"Invalid base URL format: {base_url}")
+    cleaned_href = href.strip()
 
     # Use urljoin to handle all cases
-    normalized = urljoin(base_url, href.strip())
+    return urljoin(base_url, cleaned_href)
+
+
+
+
+def normalize_url(
+    href: str,
+    base_url: str,
+    *,
+    drop_query_tracking=True,
+    sort_query=True,
+    keep_fragment=False,
+    extra_drop_params=None
+):
+    """
+    Extended URL normalizer
+
+    Parameters
+    ----------
+    href : str
+        The raw link extracted from a page.
+    base_url : str
+        The page’s canonical URL (used to resolve relative links).
+    drop_query_tracking : bool (default True)
+        Remove common tracking query parameters.
+    sort_query : bool (default True)
+        Alphabetically sort query keys for deterministic output.
+    keep_fragment : bool (default False)
+        Preserve the hash fragment (#section) if you need in-page links.
+    extra_drop_params : Iterable[str] | None
+        Additional query keys to strip (case-insensitive).
+
+    Returns
+    -------
+    str | None
+        A clean, canonical URL or None if href is empty/None.
+    """
+    if not href:
+        return None
+
+    # Resolve relative paths first
+    full_url = urljoin(base_url, href.strip())
+
+    # Parse once, edit parts, then rebuild
+    parsed = urlparse(full_url)
+
+    # ── netloc ──
+    netloc = parsed.netloc.lower()
+
+    # ── path ──
+    # Strip duplicate slashes and trailing “/” (except root)
+    path = quote(unquote(parsed.path))
+    if path.endswith('/') and path != '/':
+        path = path.rstrip('/')
+
+    # ── query ──
+    query = parsed.query
+    if query:
+        # explode, mutate, then rebuild
+        params = [(k.lower(), v) for k, v in parse_qsl(query, keep_blank_values=True)]
+
+        if drop_query_tracking:
+            default_tracking = {
+                'utm_source', 'utm_medium', 'utm_campaign', 'utm_term',
+                'utm_content', 'gclid', 'fbclid', 'ref', 'ref_src'
+            }
+            if extra_drop_params:
+                default_tracking |= {p.lower() for p in extra_drop_params}
+            params = [(k, v) for k, v in params if k not in default_tracking]
+
+        if sort_query:
+            params.sort(key=lambda kv: kv[0])
+
+        query = urlencode(params, doseq=True) if params else ''
+
+    # ── fragment ──
+    fragment = parsed.fragment if keep_fragment else ''
+
+    # Re-assemble
+    normalized = urlunparse((
+        parsed.scheme,
+        netloc,
+        path,
+        parsed.params,
+        query,
+        fragment
+    ))
+
     return normalized
 
 
@@ -2047,7 +2265,7 @@ def normalize_url_for_deep_crawl(href, base_url):
     normalized = urlunparse((
         parsed.scheme,
         netloc,
-        parsed.path.rstrip('/') or '/',  # Normalize trailing slash
+        parsed.path.rstrip('/'),  # Normalize trailing slash
         parsed.params,
         query,
         fragment
@@ -2075,7 +2293,7 @@ def efficient_normalize_url_for_deep_crawl(href, base_url):
     normalized = urlunparse((
         parsed.scheme,
         parsed.netloc.lower(),
-        parsed.path,
+        parsed.path.rstrip('/'),
         parsed.params,
         parsed.query,
         ''  # Remove fragment
@@ -2733,33 +2951,67 @@ def preprocess_html_for_schema(html_content, text_threshold=100, attr_value_thre
             # Also truncate tail text if present
             if element.tail and len(element.tail.strip()) > text_threshold:
                 element.tail = element.tail.strip()[:text_threshold] + '...'
-        
-        # 4. Find repeated patterns and keep only a few examples
-        # This is a simplistic approach - more sophisticated pattern detection could be implemented
-        pattern_elements = {}
-        for element in tree.xpath('//*[contains(@class, "")]'):
-            parent = element.getparent()
+
+        # 4. Detect duplicates and drop them in a single pass
+        seen: dict[tuple, None] = {}
+        for el in list(tree.xpath('//*[@class]')):          # snapshot once, XPath is fast
+            parent = el.getparent()
             if parent is None:
                 continue
-                
-            # Create a signature based on tag and classes
-            classes = element.get('class', '')
-            if not classes:
+
+            cls = el.get('class')
+            if not cls:
                 continue
-            signature = f"{element.tag}.{classes}"
-            
-            if signature in pattern_elements:
-                pattern_elements[signature].append(element)
+
+            # ── build signature ───────────────────────────────────────────
+            h = xxhash.xxh64()                              # stream, no big join()
+            for txt in el.itertext():
+                h.update(txt)
+            sig = (el.tag, cls, h.intdigest())             # tuple cheaper & hashable
+
+            # ── first seen? keep – else drop ─────────────
+            if sig in seen and parent is not None:
+                parent.remove(el)                           # duplicate
             else:
-                pattern_elements[signature] = [element]
+                seen[sig] = None
         
-        # Keep only 3 examples of each repeating pattern
-        for signature, elements in pattern_elements.items():
-            if len(elements) > 3:
-                # Keep the first 2 and last elements
-                for element in elements[2:-1]:
-                    if element.getparent() is not None:
-                        element.getparent().remove(element)
+        # # 4. Find repeated patterns and keep only a few examples
+        # # This is a simplistic approach - more sophisticated pattern detection could be implemented
+        # pattern_elements = {}
+        # for element in tree.xpath('//*[contains(@class, "")]'):
+        #     parent = element.getparent()
+        #     if parent is None:
+        #         continue
+                
+        #     # Create a signature based on tag and classes
+        #     classes = element.get('class', '')
+        #     if not classes:
+        #         continue
+        #     innert_text = ''.join(element.xpath('.//text()'))
+        #     innert_text_hash = xxhash.xxh64(innert_text.encode()).hexdigest()
+        #     signature = f"{element.tag}.{classes}.{innert_text_hash}"
+            
+        #     if signature in pattern_elements:
+        #         pattern_elements[signature].append(element)
+        #     else:
+        #         pattern_elements[signature] = [element]
+        
+        # # Keep only first examples of each repeating pattern
+        # for signature, elements in pattern_elements.items():
+        #     if len(elements) > 1:
+        #         # Keep the first element and remove the rest
+        #         for element in elements[1:]:
+        #             if element.getparent() is not None:
+        #                 element.getparent().remove(element)
+
+
+        # # Keep only 3 examples of each repeating pattern
+        # for signature, elements in pattern_elements.items():
+        #     if len(elements) > 3:
+        #         # Keep the first 2 and last elements
+        #         for element in elements[2:-1]:
+        #             if element.getparent() is not None:
+        #                 element.getparent().remove(element)
         
         # 5. Convert back to string
         result = etree.tostring(tree, encoding='unicode', method='html')
@@ -2772,6 +3024,393 @@ def preprocess_html_for_schema(html_content, text_threshold=100, attr_value_thre
     
     except Exception as e:
         # Fallback for parsing errors
-        return html_content[:max_size] if len(html_content) > max_size else html_content
-    
+        return html_content[:max_size] if len(html_content) > max_size else html_content    
+
+def start_colab_display_server():
+    """
+    Start virtual display server in Google Colab.
+    Raises error if not running in Colab environment.
+    """
+    # Check if running in Google Colab
+    try:
+        import google.colab
+        from google.colab import output
+        from IPython.display import IFrame, display
+    except ImportError:
+        raise RuntimeError("This function must be run in Google Colab environment.")
+    
+    import os, time, subprocess
+    
+    os.environ["DISPLAY"] = ":99"
+    
+    # Xvfb
+    xvfb = subprocess.Popen(["Xvfb", ":99", "-screen", "0", "1280x720x24"])
+    time.sleep(2)
+    
+    # minimal window manager
+    fluxbox = subprocess.Popen(["fluxbox"])
+    
+    # VNC → X
+    x11vnc = subprocess.Popen(["x11vnc",
+                              "-display", ":99",
+                              "-nopw", "-forever", "-shared",
+                              "-rfbport", "5900", "-quiet"])
+    
+    # websockify → VNC
+    novnc = subprocess.Popen(["/opt/novnc/utils/websockify/run",
+                              "6080", "localhost:5900",
+                              "--web", "/opt/novnc"])
+    
+    time.sleep(2)  # give ports a moment
+    
+    # Colab proxy url
+    url = output.eval_js("google.colab.kernel.proxyPort(6080)")
+    display(IFrame(f"{url}/vnc.html?autoconnect=true&resize=scale", width=1024, height=768))
+
+
+
+def setup_colab_environment():
+    """
+    Alternative setup using IPython magic commands
+    """
+    from IPython import get_ipython
+    ipython = get_ipython()
+    
+    print("🚀 Setting up Crawl4AI environment in Google Colab...")
+    
+    # Run the bash commands
+    ipython.run_cell_magic('bash', '', '''
+set -e
+
+echo "📦 Installing system dependencies..."
+apt-get update -y
+apt-get install -y xvfb x11vnc fluxbox websockify git
+
+echo "📥 Setting up virtual display..."
+git clone https://github.com/novnc/noVNC         /opt/novnc
+git clone https://github.com/novnc/websockify    /opt/novnc/utils/websockify
+
+pip install -q nest_asyncio google-colab
+echo "✅ Setup complete!"
+''')
+
+
+# Link Quality Scoring Functions
+def extract_page_context(page_title: str, headlines_text: str, meta_description: str, base_url: str) -> dict:
+    """
+    Extract page context for link scoring - called ONCE per page for performance.
+    Parser-agnostic function that takes pre-extracted data.
+    
+    Args:
+        page_title: Title of the page
+        headlines_text: Combined text from h1, h2, h3 elements
+        meta_description: Meta description content
+        base_url: Base URL of the page
+        
+    Returns:
+        Dictionary containing page context data for fast link scoring
+    """
+    context = {
+        'terms': set(),
+        'headlines': headlines_text or '',
+        'meta_description': meta_description or '',
+        'domain': '',
+        'is_docs_site': False
+    }
+    
+    try:
+        from urllib.parse import urlparse
+        parsed = urlparse(base_url)
+        context['domain'] = parsed.netloc.lower()
+        
+        # Check if this is a documentation/reference site
+        context['is_docs_site'] = any(indicator in context['domain'] 
+                                    for indicator in ['docs.', 'api.', 'developer.', 'reference.'])
+        
+        # Create term set for fast intersection (performance optimization)
+        all_text = ((page_title or '') + ' ' + context['headlines'] + ' ' + context['meta_description']).lower()
+        # Simple tokenization - fast and sufficient for scoring
+        context['terms'] = set(word.strip('.,!?;:"()[]{}') 
+                             for word in all_text.split() 
+                             if len(word.strip('.,!?;:"()[]{}')) > 2)
+                             
+    except Exception:
+        # Fail gracefully - return empty context
+        pass
+    
+    return context
+
+
+def calculate_link_intrinsic_score(
+    link_text: str, 
+    url: str, 
+    title_attr: str, 
+    class_attr: str, 
+    rel_attr: str, 
+    page_context: dict
+) -> float:
+    """
+    Ultra-fast link quality scoring using only provided data (no DOM access needed).
+    Parser-agnostic function.
+    
+    Args:
+        link_text: Text content of the link
+        url: Link URL
+        title_attr: Title attribute of the link
+        class_attr: Class attribute of the link
+        rel_attr: Rel attribute of the link
+        page_context: Pre-computed page context from extract_page_context()
+        
+    Returns:
+        Quality score (0.0 - 10.0), higher is better
+    """
+    score = 0.0
+    
+    try:
+        # 1. ATTRIBUTE QUALITY (string analysis - very fast)
+        if title_attr and len(title_attr.strip()) > 3:
+            score += 1.0
+            
+        class_str = (class_attr or '').lower()
+        # Navigation/important classes boost score
+        if any(nav_class in class_str for nav_class in ['nav', 'menu', 'primary', 'main', 'important']):
+            score += 1.5
+        # Marketing/ad classes reduce score  
+        if any(bad_class in class_str for bad_class in ['ad', 'sponsor', 'track', 'promo', 'banner']):
+            score -= 1.0
+            
+        rel_str = (rel_attr or '').lower()
+        # Semantic rel values
+        if any(good_rel in rel_str for good_rel in ['canonical', 'next', 'prev', 'chapter']):
+            score += 1.0
+        if any(bad_rel in rel_str for bad_rel in ['nofollow', 'sponsored', 'ugc']):
+            score -= 0.5
+            
+        # 2. URL STRUCTURE QUALITY (string operations - very fast)
+        url_lower = url.lower()
+        
+        # High-value path patterns
+        if any(good_path in url_lower for good_path in ['/docs/', '/api/', '/guide/', '/tutorial/', '/reference/', '/manual/']):
+            score += 2.0
+        elif any(medium_path in url_lower for medium_path in ['/blog/', '/article/', '/post/', '/news/']):
+            score += 1.0
+            
+        # Penalize certain patterns
+        if any(bad_path in url_lower for bad_path in ['/admin/', '/login/', '/cart/', '/checkout/', '/track/', '/click/']):
+            score -= 1.5
+            
+        # URL depth (shallow URLs often more important)
+        url_depth = url.count('/') - 2  # Subtract protocol and domain
+        if url_depth <= 2:
+            score += 1.0
+        elif url_depth > 5:
+            score -= 0.5
+            
+        # HTTPS bonus
+        if url.startswith('https://'):
+            score += 0.5
+            
+        # 3. TEXT QUALITY (string analysis - very fast)
+        if link_text:
+            text_clean = link_text.strip()
+            if len(text_clean) > 3:
+                score += 1.0
+                
+            # Multi-word links are usually more descriptive
+            word_count = len(text_clean.split())
+            if word_count >= 2:
+                score += 0.5
+            if word_count >= 4:
+                score += 0.5
+                
+            # Avoid generic link text
+            generic_texts = ['click here', 'read more', 'more info', 'link', 'here']
+            if text_clean.lower() in generic_texts:
+                score -= 1.0
+                
+        # 4. CONTEXTUAL RELEVANCE (pre-computed page terms - very fast)
+        if page_context.get('terms') and link_text:
+            link_words = set(word.strip('.,!?;:"()[]{}').lower() 
+                           for word in link_text.split() 
+                           if len(word.strip('.,!?;:"()[]{}')) > 2)
+            
+            if link_words:
+                # Calculate word overlap ratio
+                overlap = len(link_words & page_context['terms'])
+                if overlap > 0:
+                    relevance_ratio = overlap / min(len(link_words), 10)  # Cap to avoid over-weighting
+                    score += relevance_ratio * 2.0  # Up to 2 points for relevance
+                    
+        # 5. DOMAIN CONTEXT BONUSES (very fast string checks)
+        if page_context.get('is_docs_site', False):
+            # Documentation sites: prioritize internal navigation
+            if link_text and any(doc_keyword in link_text.lower() 
+                               for doc_keyword in ['api', 'reference', 'guide', 'tutorial', 'example']):
+                score += 1.0
+                
+    except Exception:
+        # Fail gracefully - return minimal score
+        score = 0.5
+        
+    # Ensure score is within reasonable bounds
+    return max(0.0, min(score, 10.0))
+
+
+def calculate_total_score(
+    intrinsic_score: Optional[float] = None,
+    contextual_score: Optional[float] = None,
+    score_links_enabled: bool = False,
+    query_provided: bool = False
+) -> float:
+    """
+    Calculate combined total score from intrinsic and contextual scores with smart fallbacks.
+    
+    Args:
+        intrinsic_score: Quality score based on URL structure, text, and context (0-10)
+        contextual_score: BM25 relevance score based on query and head content (0-1 typically)
+        score_links_enabled: Whether link scoring is enabled
+        query_provided: Whether a query was provided for contextual scoring
+        
+    Returns:
+        Combined total score (0-10 scale)
+        
+    Scoring Logic:
+        - No scoring: return 5.0 (neutral score)
+        - Only intrinsic: return normalized intrinsic score
+        - Only contextual: return contextual score scaled to 10
+        - Both: weighted combination (70% intrinsic, 30% contextual scaled)
+    """
+    # Case 1: No scoring enabled at all
+    if not score_links_enabled:
+        return 5.0  # Neutral score - all links treated equally
+    
+    # Normalize scores to handle None values
+    intrinsic = intrinsic_score if intrinsic_score is not None else 0.0
+    contextual = contextual_score if contextual_score is not None else 0.0
+    
+    # Case 2: Only intrinsic scoring (no query provided or no head extraction)
+    if not query_provided or contextual_score is None:
+        # Use intrinsic score directly (already 0-10 scale)
+        return max(0.0, min(intrinsic, 10.0))
+    
+    # Case 3: Both intrinsic and contextual scores available
+    # Scale contextual score (typically 0-1) to 0-10 range
+    contextual_scaled = min(contextual * 10.0, 10.0)
+    
+    # Weighted combination: 70% intrinsic (structure/content quality) + 30% contextual (query relevance)
+    # This gives more weight to link quality while still considering relevance
+    total = (intrinsic * 0.7) + (contextual_scaled * 0.3)
+    
+    return max(0.0, min(total, 10.0))
+
+
+# Embedding utilities
+async def get_text_embeddings(
+    texts: List[str], 
+    llm_config: Optional[Dict] = None,
+    model_name: str = "sentence-transformers/all-MiniLM-L6-v2",
+    batch_size: int = 32
+) -> np.ndarray:
+    """
+    Compute embeddings for a list of texts using specified model.
+    
+    Args:
+        texts: List of texts to embed
+        llm_config: Optional LLM configuration for API-based embeddings
+        model_name: Model name (used when llm_config is None)
+        batch_size: Batch size for processing
+        
+    Returns:
+        numpy array of embeddings
+    """
+    import numpy as np
+    
+    if not texts:
+        return np.array([])
+    
+    # If LLMConfig provided, use litellm for embeddings
+    if llm_config is not None:
+        from litellm import aembedding
+        
+        # Get embedding model from config or use default
+        embedding_model = llm_config.get('provider', 'text-embedding-3-small')
+        api_base = llm_config.get('base_url', llm_config.get('api_base'))
+        
+        # Prepare kwargs
+        kwargs = {
+            'model': embedding_model,
+            'input': texts,
+            'api_key': llm_config.get('api_token', llm_config.get('api_key'))
+        }
+        
+        if api_base:
+            kwargs['api_base'] = api_base
+            
+        # Handle OpenAI-compatible endpoints
+        if api_base and 'openai/' not in embedding_model:
+            kwargs['model'] = f"openai/{embedding_model}"
+        
+        # Get embeddings
+        response = await aembedding(**kwargs)
+        
+        # Extract embeddings from response
+        embeddings = []
+        for item in response.data:
+            embeddings.append(item['embedding'])
+            
+        return np.array(embeddings)
+    
+    # Default: use sentence-transformers
+    else:
+        # Lazy load to avoid importing heavy libraries unless needed
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError:
+            raise ImportError(
+                "sentence-transformers is required for local embeddings. "
+                "Install it with: pip install 'crawl4ai[transformer]' or pip install sentence-transformers"
+            )
+        
+        # Cache the model in function attribute to avoid reloading
+        if not hasattr(get_text_embeddings, '_models'):
+            get_text_embeddings._models = {}
+        
+        if model_name not in get_text_embeddings._models:
+            get_text_embeddings._models[model_name] = SentenceTransformer(model_name)
+        
+        encoder = get_text_embeddings._models[model_name]
+        
+        # Batch encode for efficiency
+        embeddings = encoder.encode(
+            texts,
+            batch_size=batch_size,
+            show_progress_bar=False,
+            convert_to_numpy=True
+        )
+        
+        return embeddings
+
+
+def get_text_embeddings_sync(
+    texts: List[str],
+    llm_config: Optional[Dict] = None,
+    model_name: str = "sentence-transformers/all-MiniLM-L6-v2",
+    batch_size: int = 32
+) -> np.ndarray:
+    """Synchronous wrapper for get_text_embeddings"""
+    import numpy as np
+    return asyncio.run(get_text_embeddings(texts, llm_config, model_name, batch_size))
+
+
+def cosine_similarity(vec1: np.ndarray, vec2: np.ndarray) -> float:
+    """Calculate cosine similarity between two vectors"""
+    import numpy as np
+    dot_product = np.dot(vec1, vec2)
+    norm_product = np.linalg.norm(vec1) * np.linalg.norm(vec2)
+    return float(dot_product / norm_product) if norm_product != 0 else 0.0
+
+
+def cosine_distance(vec1: np.ndarray, vec2: np.ndarray) -> float:
+    """Calculate cosine distance (1 - similarity) between two vectors"""
+    return 1 - cosine_similarity(vec1, vec2)
 

deploy/docker/.llm.env.example

@@ -5,4 +5,9 @@ ANTHROPIC_API_KEY=your_anthropic_key_here
 GROQ_API_KEY=your_groq_key_here
 TOGETHER_API_KEY=your_together_key_here
 MISTRAL_API_KEY=your_mistral_key_here
-GEMINI_API_TOKEN=your_gemini_key_here
+GEMINI_API_TOKEN=your_gemini_key_here
+
+# Optional: Override the default LLM provider
+# Examples: "openai/gpt-4", "anthropic/claude-3-opus", "deepseek/chat", etc.
+# If not set, uses the provider specified in config.yml (default: openai/gpt-4o-mini)
+# LLM_PROVIDER=anthropic/claude-3-opus

deploy/docker/api.py

@@ -1,8 +1,11 @@
 import os
 import json
 import asyncio
-from typing import List, Tuple
+from typing import List, Tuple, Dict
 from functools import partial
+from uuid import uuid4
+from datetime import datetime
+from base64 import b64encode
 
 import logging
 from typing import Optional, AsyncGenerator
@@ -37,11 +40,24 @@ from utils import (
     get_base_url,
     is_task_id,
     should_cleanup_task,
-    decode_redis_hash
+    decode_redis_hash,
+    get_llm_api_key,
+    validate_llm_provider
 )
 
+import psutil, time
+
 logger = logging.getLogger(__name__)
 
+# --- Helper to get memory ---
+def _get_memory_mb():
+    try:
+        return psutil.Process().memory_info().rss / (1024 * 1024)
+    except Exception as e:
+        logger.warning(f"Could not get memory info: {e}")
+        return None
+
+
 async def handle_llm_qa(
     url: str,
     query: str,
@@ -49,6 +65,8 @@ async def handle_llm_qa(
 ) -> str:
     """Process QA using LLM with crawled content as context."""
     try:
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
         # Extract base URL by finding last '?q=' occurrence
         last_q_index = url.rfind('?q=')
         if last_q_index != -1:
@@ -62,7 +80,7 @@ async def handle_llm_qa(
                     status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                     detail=result.error_message
                 )
-            content = result.markdown.fit_markdown
+            content = result.markdown.fit_markdown or result.markdown.raw_markdown
 
         # Create prompt and get LLM response
         prompt = f"""Use the following content as context to answer the question.
@@ -73,10 +91,12 @@ async def handle_llm_qa(
 
     Answer:"""
 
+        # api_token=os.environ.get(config["llm"].get("api_key_env", ""))
+
         response = perform_completion_with_backoff(
             provider=config["llm"]["provider"],
             prompt_with_variables=prompt,
-            api_token=os.environ.get(config["llm"].get("api_key_env", ""))
+            api_token=get_llm_api_key(config)
         )
 
         return response.choices[0].message.content
@@ -94,19 +114,23 @@ async def process_llm_extraction(
     url: str,
     instruction: str,
     schema: Optional[str] = None,
-    cache: str = "0"
+    cache: str = "0",
+    provider: Optional[str] = None
 ) -> None:
     """Process LLM extraction in background."""
     try:
-        # If config['llm'] has api_key then ignore the api_key_env
-        api_key = ""
-        if "api_key" in config["llm"]:
-            api_key = config["llm"]["api_key"]
-        else:
-            api_key = os.environ.get(config["llm"].get("api_key_env", None), "")
+        # Validate provider
+        is_valid, error_msg = validate_llm_provider(config, provider)
+        if not is_valid:
+            await redis.hset(f"task:{task_id}", mapping={
+                "status": TaskStatus.FAILED,
+                "error": error_msg
+            })
+            return
+        api_key = get_llm_api_key(config, provider)
         llm_strategy = LLMExtractionStrategy(
             llm_config=LLMConfig(
-                provider=config["llm"]["provider"],
+                provider=provider or config["llm"]["provider"],
                 api_token=api_key
             ),
             instruction=instruction,
@@ -153,10 +177,19 @@ async def handle_markdown_request(
     filter_type: FilterType,
     query: Optional[str] = None,
     cache: str = "0",
-    config: Optional[dict] = None
+    config: Optional[dict] = None,
+    provider: Optional[str] = None
 ) -> str:
     """Handle markdown generation requests."""
     try:
+        # Validate provider if using LLM filter
+        if filter_type == FilterType.LLM:
+            is_valid, error_msg = validate_llm_provider(config, provider)
+            if not is_valid:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=error_msg
+                )
         decoded_url = unquote(url)
         if not decoded_url.startswith(('http://', 'https://')):
             decoded_url = 'https://' + decoded_url
@@ -169,8 +202,8 @@ async def handle_markdown_request(
                 FilterType.BM25: BM25ContentFilter(user_query=query or ""),
                 FilterType.LLM: LLMContentFilter(
                     llm_config=LLMConfig(
-                        provider=config["llm"]["provider"],
-                        api_token=os.environ.get(config["llm"].get("api_key_env", None), ""),
+                        provider=provider or config["llm"]["provider"],
+                        api_token=get_llm_api_key(config, provider),
                     ),
                     instruction=query or "Extract main content"
                 )
@@ -214,7 +247,8 @@ async def handle_llm_request(
     query: Optional[str] = None,
     schema: Optional[str] = None,
     cache: str = "0",
-    config: Optional[dict] = None
+    config: Optional[dict] = None,
+    provider: Optional[str] = None
 ) -> JSONResponse:
     """Handle LLM extraction requests."""
     base_url = get_base_url(request)
@@ -244,7 +278,8 @@ async def handle_llm_request(
             schema,
             cache,
             base_url,
-            config
+            config,
+            provider
         )
 
     except Exception as e:
@@ -259,7 +294,9 @@ async def handle_llm_request(
 async def handle_task_status(
     redis: aioredis.Redis,
     task_id: str,
-    base_url: str
+    base_url: str,
+    *,
+    keep: bool = False
 ) -> JSONResponse:
     """Handle task status check requests."""
     task = await redis.hgetall(f"task:{task_id}")
@@ -273,7 +310,7 @@ async def handle_task_status(
     response = create_task_response(task, task_id, base_url)
 
     if task["status"] in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
-        if should_cleanup_task(task["created_at"]):
+        if not keep and should_cleanup_task(task["created_at"]):
             await redis.delete(f"task:{task_id}")
 
     return JSONResponse(response)
@@ -286,7 +323,8 @@ async def create_new_task(
     schema: Optional[str],
     cache: str,
     base_url: str,
-    config: dict
+    config: dict,
+    provider: Optional[str] = None
 ) -> JSONResponse:
     """Create and initialize a new task."""
     decoded_url = unquote(input_path)
@@ -310,7 +348,8 @@ async def create_new_task(
         decoded_url,
         query,
         schema,
-        cache
+        cache,
+        provider
     )
 
     return JSONResponse({
@@ -351,7 +390,12 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)
     try:
         async for result in results_gen:
             try:
+                server_memory_mb = _get_memory_mb()
                 result_dict = result.model_dump()
+                result_dict['server_memory_mb'] = server_memory_mb
+                # If PDF exists, encode it to base64
+                if result_dict.get('pdf') is not None:
+                    result_dict['pdf'] = b64encode(result_dict['pdf']).decode('utf-8')
                 logger.info(f"Streaming result for {result_dict.get('url', 'unknown')}")
                 data = json.dumps(result_dict, default=datetime_handler) + "\n"
                 yield data.encode('utf-8')
@@ -365,10 +409,11 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)
     except asyncio.CancelledError:
         logger.warning("Client disconnected during streaming")
     finally:
-        try:
-            await crawler.close()
-        except Exception as e:
-            logger.error(f"Crawler cleanup error: {e}")
+        # try:
+        #     await crawler.close()
+        # except Exception as e:
+        #     logger.error(f"Crawler cleanup error: {e}")
+        pass
 
 async def handle_crawl_request(
     urls: List[str],
@@ -377,7 +422,13 @@ async def handle_crawl_request(
     config: dict
 ) -> dict:
     """Handle non-streaming crawl requests."""
+    start_mem_mb = _get_memory_mb() # <--- Get memory before
+    start_time = time.time()
+    mem_delta_mb = None
+    peak_mem_mb = start_mem_mb
+    
     try:
+        urls = [('https://' + url) if not url.startswith(('http://', 'https://')) else url for url in urls]
         browser_config = BrowserConfig.load(browser_config)
         crawler_config = CrawlerRunConfig.load(crawler_config)
 
@@ -385,27 +436,77 @@ async def handle_crawl_request(
             memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
             rate_limiter=RateLimiter(
                 base_delay=tuple(config["crawler"]["rate_limiter"]["base_delay"])
-            )
+            ) if config["crawler"]["rate_limiter"]["enabled"] else None
         )
+        
+        from crawler_pool import get_crawler
+        crawler = await get_crawler(browser_config)
 
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            results = []
-            func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
-            partial_func = partial(func, 
-                                   urls[0] if len(urls) == 1 else urls, 
-                                   config=crawler_config, 
-                                   dispatcher=dispatcher)
-            results = await partial_func()
-            return {
-                "success": True,
-                "results": [result.model_dump() for result in results]
-            }
+        # crawler: AsyncWebCrawler = AsyncWebCrawler(config=browser_config)
+        # await crawler.start()
+        
+        base_config = config["crawler"]["base_config"]
+        # Iterate on key-value pairs in global_config then use haseattr to set them 
+        for key, value in base_config.items():
+            if hasattr(crawler_config, key):
+                setattr(crawler_config, key, value)
+
+        results = []
+        func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
+        partial_func = partial(func, 
+                                urls[0] if len(urls) == 1 else urls, 
+                                config=crawler_config, 
+                                dispatcher=dispatcher)
+        results = await partial_func()
+
+        # await crawler.close()
+        
+        end_mem_mb = _get_memory_mb() # <--- Get memory after
+        end_time = time.time()
+        
+        if start_mem_mb is not None and end_mem_mb is not None:
+            mem_delta_mb = end_mem_mb - start_mem_mb # <--- Calculate delta
+            peak_mem_mb = max(peak_mem_mb if peak_mem_mb else 0, end_mem_mb) # <--- Get peak memory
+        logger.info(f"Memory usage: Start: {start_mem_mb} MB, End: {end_mem_mb} MB, Delta: {mem_delta_mb} MB, Peak: {peak_mem_mb} MB")
+
+        # Process results to handle PDF bytes
+        processed_results = []
+        for result in results:
+            result_dict = result.model_dump()
+            # If PDF exists, encode it to base64
+            if result_dict.get('pdf') is not None:
+                result_dict['pdf'] = b64encode(result_dict['pdf']).decode('utf-8')
+            processed_results.append(result_dict)
+            
+        return {
+            "success": True,
+            "results": processed_results,
+            "server_processing_time_s": end_time - start_time,
+            "server_memory_delta_mb": mem_delta_mb,
+            "server_peak_memory_mb": peak_mem_mb
+        }
 
     except Exception as e:
         logger.error(f"Crawl error: {str(e)}", exc_info=True)
+        if 'crawler' in locals() and crawler.ready: # Check if crawler was initialized and started
+            #  try:
+            #      await crawler.close()
+            #  except Exception as close_e:
+            #       logger.error(f"Error closing crawler during exception handling: {close_e}")
+            logger.error(f"Error closing crawler during exception handling: {str(e)}")
+
+        # Measure memory even on error if possible
+        end_mem_mb_error = _get_memory_mb()
+        if start_mem_mb is not None and end_mem_mb_error is not None:
+            mem_delta_mb = end_mem_mb_error - start_mem_mb
+
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=str(e)
+            detail=json.dumps({ # Send structured error
+                "error": str(e),
+                "server_memory_delta_mb": mem_delta_mb,
+                "server_peak_memory_mb": max(peak_mem_mb if peak_mem_mb else 0, end_mem_mb_error or 0)
+            })
         )
 
 async def handle_stream_crawl_request(
@@ -417,9 +518,11 @@ async def handle_stream_crawl_request(
     """Handle streaming crawl requests."""
     try:
         browser_config = BrowserConfig.load(browser_config)
-        browser_config.verbose = True
+        # browser_config.verbose = True # Set to False or remove for production stress testing
+        browser_config.verbose = False
         crawler_config = CrawlerRunConfig.load(crawler_config)
         crawler_config.scraping_strategy = LXMLWebScrapingStrategy()
+        crawler_config.stream = True
 
         dispatcher = MemoryAdaptiveDispatcher(
             memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
@@ -428,8 +531,11 @@ async def handle_stream_crawl_request(
             )
         )
 
-        crawler = AsyncWebCrawler(config=browser_config)
-        await crawler.start()
+        from crawler_pool import get_crawler
+        crawler = await get_crawler(browser_config)
+
+        # crawler = AsyncWebCrawler(config=browser_config)
+        # await crawler.start()
 
         results_gen = await crawler.arun_many(
             urls=urls,
@@ -440,10 +546,60 @@ async def handle_stream_crawl_request(
         return crawler, results_gen
 
     except Exception as e:
-        if 'crawler' in locals():
-            await crawler.close()
+        # Make sure to close crawler if started during an error here
+        if 'crawler' in locals() and crawler.ready:
+            #  try:
+            #       await crawler.close()
+            #  except Exception as close_e:
+            #       logger.error(f"Error closing crawler during stream setup exception: {close_e}")
+            logger.error(f"Error closing crawler during stream setup exception: {str(e)}")
         logger.error(f"Stream crawl error: {str(e)}", exc_info=True)
+        # Raising HTTPException here will prevent streaming response
         raise HTTPException(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail=str(e)
-        )
+        )
+        
+async def handle_crawl_job(
+    redis,
+    background_tasks: BackgroundTasks,
+    urls: List[str],
+    browser_config: Dict,
+    crawler_config: Dict,
+    config: Dict,
+) -> Dict:
+    """
+    Fire-and-forget version of handle_crawl_request.
+    Creates a task in Redis, runs the heavy work in a background task,
+    lets /crawl/job/{task_id} polling fetch the result.
+    """
+    task_id = f"crawl_{uuid4().hex[:8]}"
+    await redis.hset(f"task:{task_id}", mapping={
+        "status": TaskStatus.PROCESSING,         # <-- keep enum values consistent
+        "created_at": datetime.utcnow().isoformat(),
+        "url": json.dumps(urls),                 # store list as JSON string
+        "result": "",
+        "error": "",
+    })
+
+    async def _runner():
+        try:
+            result = await handle_crawl_request(
+                urls=urls,
+                browser_config=browser_config,
+                crawler_config=crawler_config,
+                config=config,
+            )
+            await redis.hset(f"task:{task_id}", mapping={
+                "status": TaskStatus.COMPLETED,
+                "result": json.dumps(result),
+            })
+            await asyncio.sleep(5)  # Give Redis time to process the update
+        except Exception as exc:
+            await redis.hset(f"task:{task_id}", mapping={
+                "status": TaskStatus.FAILED,
+                "error": str(exc),
+            })
+
+    background_tasks.add_task(_runner)
+    return {"task_id": task_id}

deploy/docker/config.yml

@@ -3,8 +3,9 @@ app:
   title: "Crawl4AI API"
   version: "1.0.0"
   host: "0.0.0.0"
-  port: 8020
-  reload: True
+  port: 11234
+  reload: False
+  workers: 1
   timeout_keep_alive: 300
 
 # Default LLM Configuration
@@ -50,12 +51,31 @@ security:
 
 # Crawler Configuration
 crawler:
+  base_config:
+    simulate_user: true
   memory_threshold_percent: 95.0
   rate_limiter:
+    enabled: true
     base_delay: [1.0, 2.0]
   timeouts:
     stream_init: 30.0  # Timeout for stream initialization
     batch_process: 300.0  # Timeout for batch processing
+  pool:
+    max_pages: 40                          # ← GLOBAL_SEM permits
+    idle_ttl_sec: 1800                     # ← 30 min janitor cutoff
+  browser:
+    kwargs:
+      headless: true
+      text_mode: true
+    extra_args:
+      # - "--single-process"
+      - "--no-sandbox"
+      - "--disable-dev-shm-usage"
+      - "--disable-gpu"
+      - "--disable-software-rasterizer"
+      - "--disable-web-security"
+      - "--allow-insecure-localhost"
+      - "--ignore-certificate-errors"
 
 # Logging Configuration
 logging:

deploy/docker/crawler_pool.py

@@ -0,0 +1,60 @@
+# crawler_pool.py  (new file)
+import asyncio, json, hashlib, time, psutil
+from contextlib import suppress
+from typing import Dict
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+from typing import Dict
+from utils import load_config 
+
+CONFIG = load_config()
+
+POOL: Dict[str, AsyncWebCrawler] = {}
+LAST_USED: Dict[str, float] = {}
+LOCK = asyncio.Lock()
+
+MEM_LIMIT  = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0)   # % RAM – refuse new browsers above this
+IDLE_TTL  = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800)   # close if unused for 30 min
+
+def _sig(cfg: BrowserConfig) -> str:
+    payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":"))
+    return hashlib.sha1(payload.encode()).hexdigest()
+
+async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
+    try:
+        sig = _sig(cfg)
+        async with LOCK:
+            if sig in POOL:
+                LAST_USED[sig] = time.time();  
+                return POOL[sig]
+            if psutil.virtual_memory().percent >= MEM_LIMIT:
+                raise MemoryError("RAM pressure – new browser denied")
+            crawler = AsyncWebCrawler(config=cfg, thread_safe=False)
+            await crawler.start()
+            POOL[sig] = crawler; LAST_USED[sig] = time.time()
+            return crawler
+    except MemoryError as e:
+        raise MemoryError(f"RAM pressure – new browser denied: {e}")
+    except Exception as e:
+        raise RuntimeError(f"Failed to start browser: {e}")
+    finally:
+        if sig in POOL:
+            LAST_USED[sig] = time.time()
+        else:
+            # If we failed to start the browser, we should remove it from the pool
+            POOL.pop(sig, None)
+            LAST_USED.pop(sig, None)
+        # If we failed to start the browser, we should remove it from the pool
+async def close_all():
+    async with LOCK:
+        await asyncio.gather(*(c.close() for c in POOL.values()), return_exceptions=True)
+        POOL.clear(); LAST_USED.clear()
+
+async def janitor():
+    while True:
+        await asyncio.sleep(60)
+        now = time.time()
+        async with LOCK:
+            for sig, crawler in list(POOL.items()):
+                if now - LAST_USED[sig] > IDLE_TTL:
+                    with suppress(Exception): await crawler.close()
+                    POOL.pop(sig, None); LAST_USED.pop(sig, None)

deploy/docker/job.py

@@ -0,0 +1,101 @@
+"""
+Job endpoints (enqueue + poll) for long-running LL​M extraction and raw crawl.
+Relies on the existing Redis task helpers in api.py
+"""
+
+from typing import Dict, Optional, Callable
+from fastapi import APIRouter, BackgroundTasks, Depends, Request
+from pydantic import BaseModel, HttpUrl
+
+from api import (
+    handle_llm_request,
+    handle_crawl_job,
+    handle_task_status,
+)
+
+# ------------- dependency placeholders -------------
+_redis = None        # will be injected from server.py
+_config = None
+_token_dep: Callable = lambda: None  # dummy until injected
+
+# public router
+router = APIRouter()
+
+
+# === init hook called by server.py =========================================
+def init_job_router(redis, config, token_dep) -> APIRouter:
+    """Inject shared singletons and return the router for mounting."""
+    global _redis, _config, _token_dep
+    _redis, _config, _token_dep = redis, config, token_dep
+    return router
+
+
+# ---------- payload models --------------------------------------------------
+class LlmJobPayload(BaseModel):
+    url:    HttpUrl
+    q:      str
+    schema: Optional[str] = None
+    cache:  bool = False
+    provider: Optional[str] = None
+
+
+class CrawlJobPayload(BaseModel):
+    urls:           list[HttpUrl]
+    browser_config: Dict = {}
+    crawler_config: Dict = {}
+
+
+# ---------- LL​M job ---------------------------------------------------------
+@router.post("/llm/job", status_code=202)
+async def llm_job_enqueue(
+        payload: LlmJobPayload,
+        background_tasks: BackgroundTasks,
+        request: Request,
+        _td: Dict = Depends(lambda: _token_dep()),   # late-bound dep
+):
+    return await handle_llm_request(
+        _redis,
+        background_tasks,
+        request,
+        str(payload.url),
+        query=payload.q,
+        schema=payload.schema,
+        cache=payload.cache,
+        config=_config,
+        provider=payload.provider,
+    )
+
+
+@router.get("/llm/job/{task_id}")
+async def llm_job_status(
+    request: Request,
+    task_id: str,
+    _td: Dict = Depends(lambda: _token_dep())
+):
+    return await handle_task_status(_redis, task_id)
+
+
+# ---------- CRAWL job -------------------------------------------------------
+@router.post("/crawl/job", status_code=202)
+async def crawl_job_enqueue(
+        payload: CrawlJobPayload,
+        background_tasks: BackgroundTasks,
+        _td: Dict = Depends(lambda: _token_dep()),
+):
+    return await handle_crawl_job(
+        _redis,
+        background_tasks,
+        [str(u) for u in payload.urls],
+        payload.browser_config,
+        payload.crawler_config,
+        config=_config,
+    )
+
+
+@router.get("/crawl/job/{task_id}")
+async def crawl_job_status(
+    request: Request,
+    task_id: str,
+    _td: Dict = Depends(lambda: _token_dep())
+):
+    return await handle_task_status(_redis, task_id, base_url=str(request.base_url))

deploy/docker/mcp_bridge.py

@@ -0,0 +1,252 @@
+# deploy/docker/mcp_bridge.py
+
+from __future__ import annotations
+import inspect, json, re, anyio
+from contextlib import suppress
+from typing import Any, Callable, Dict, List, Tuple
+import httpx
+
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException
+from fastapi.responses import JSONResponse
+from fastapi import Request
+from sse_starlette.sse import EventSourceResponse
+from pydantic import BaseModel
+from mcp.server.sse import SseServerTransport
+
+import mcp.types as t
+from mcp.server.lowlevel.server import Server, NotificationOptions
+from mcp.server.models import InitializationOptions
+
+# ── opt‑in decorators ───────────────────────────────────────────
+def mcp_resource(name: str | None = None):
+    def deco(fn):
+        fn.__mcp_kind__, fn.__mcp_name__ = "resource", name
+        return fn
+    return deco
+
+def mcp_template(name: str | None = None):
+    def deco(fn):
+        fn.__mcp_kind__, fn.__mcp_name__ = "template", name
+        return fn
+    return deco
+
+def mcp_tool(name: str | None = None):
+    def deco(fn):
+        fn.__mcp_kind__, fn.__mcp_name__ = "tool", name
+        return fn
+    return deco
+
+# ── HTTP‑proxy helper for FastAPI endpoints ─────────────────────
+def _make_http_proxy(base_url: str, route):
+    method = list(route.methods - {"HEAD", "OPTIONS"})[0]
+    async def proxy(**kwargs):
+        # replace `/items/{id}` style params first
+        path = route.path
+        for k, v in list(kwargs.items()):
+            placeholder = "{" + k + "}"
+            if placeholder in path:
+                path = path.replace(placeholder, str(v))
+                kwargs.pop(k)
+        url = base_url.rstrip("/") + path
+
+        async with httpx.AsyncClient() as client:
+            try:
+                r = (
+                    await client.get(url, params=kwargs)
+                    if method == "GET"
+                    else await client.request(method, url, json=kwargs)
+                )
+                r.raise_for_status()
+                return r.text if method == "GET" else r.json()
+            except httpx.HTTPStatusError as e:
+                # surface FastAPI error details instead of plain 500
+                raise HTTPException(e.response.status_code, e.response.text)
+    return proxy
+
+# ── main entry point ────────────────────────────────────────────
+def attach_mcp(
+    app: FastAPI,
+    *,                          # keyword‑only
+    base: str = "/mcp",
+    name: str | None = None,
+    base_url: str,              # eg. "http://127.0.0.1:8020"
+) -> None:
+    """Call once after all routes are declared to expose WS+SSE MCP endpoints."""
+    server_name = name or app.title or "FastAPI-MCP"
+    mcp = Server(server_name)
+
+    # tools: Dict[str, Callable] = {}
+    tools: Dict[str, Tuple[Callable, Callable]] = {}
+    resources: Dict[str, Callable] = {}
+    templates: Dict[str, Callable] = {}
+
+    # register decorated FastAPI routes
+    for route in app.routes:
+        fn = getattr(route, "endpoint", None)
+        kind = getattr(fn, "__mcp_kind__", None)
+        if not kind:
+            continue
+
+        key = fn.__mcp_name__ or re.sub(r"[/{}}]", "_", route.path).strip("_")
+
+        # if kind == "tool":
+        #     tools[key] = _make_http_proxy(base_url, route)
+        if kind == "tool":
+            proxy = _make_http_proxy(base_url, route)
+            tools[key] = (proxy, fn)
+            continue
+        if kind == "resource":
+            resources[key] = fn
+        if kind == "template":
+            templates[key] = fn
+
+    # helpers for JSON‑Schema
+    def _schema(model: type[BaseModel] | None) -> dict:
+        return {"type": "object"} if model is None else model.model_json_schema()
+
+    def _body_model(fn: Callable) -> type[BaseModel] | None:
+        for p in inspect.signature(fn).parameters.values():
+            a = p.annotation
+            if inspect.isclass(a) and issubclass(a, BaseModel):
+                return a
+        return None
+
+    # MCP handlers
+    @mcp.list_tools()
+    async def _list_tools() -> List[t.Tool]:
+        out = []
+        for k, (proxy, orig_fn) in tools.items():
+            desc   = getattr(orig_fn, "__mcp_description__", None) or inspect.getdoc(orig_fn) or ""
+            schema = getattr(orig_fn, "__mcp_schema__", None) or _schema(_body_model(orig_fn))
+            out.append(
+                t.Tool(name=k, description=desc, inputSchema=schema)
+            )
+        return out
+             
+
+    @mcp.call_tool()
+    async def _call_tool(name: str, arguments: Dict | None) -> List[t.TextContent]:
+        if name not in tools:
+            raise HTTPException(404, "tool not found")
+        
+        proxy, _ = tools[name]
+        try:
+            res = await proxy(**(arguments or {}))
+        except HTTPException as exc:
+            # map server‑side errors into MCP "text/error" payloads
+            err = {"error": exc.status_code, "detail": exc.detail}
+            return [t.TextContent(type = "text", text=json.dumps(err))]
+        return [t.TextContent(type = "text", text=json.dumps(res, default=str))]
+
+    @mcp.list_resources()
+    async def _list_resources() -> List[t.Resource]:
+        return [
+            t.Resource(name=k, description=inspect.getdoc(f) or "", mime_type="application/json")
+            for k, f in resources.items()
+        ]
+
+    @mcp.read_resource()
+    async def _read_resource(name: str) -> List[t.TextContent]:
+        if name not in resources:
+            raise HTTPException(404, "resource not found")
+        res = resources[name]()
+        return [t.TextContent(type = "text", text=json.dumps(res, default=str))]
+
+    @mcp.list_resource_templates()
+    async def _list_templates() -> List[t.ResourceTemplate]:
+        return [
+            t.ResourceTemplate(
+                name=k,
+                description=inspect.getdoc(f) or "",
+                parameters={
+                    p: {"type": "string"} for p in _path_params(app, f)
+                },
+            )
+            for k, f in templates.items()
+        ]
+
+    init_opts = InitializationOptions(
+        server_name=server_name,
+        server_version="0.1.0",
+        capabilities=mcp.get_capabilities(
+            notification_options=NotificationOptions(),
+            experimental_capabilities={},
+        ),
+    )
+
+    # ── WebSocket transport ────────────────────────────────────
+    @app.websocket_route(f"{base}/ws")
+    async def _ws(ws: WebSocket):
+        await ws.accept()
+        c2s_send, c2s_recv = anyio.create_memory_object_stream(100)
+        s2c_send, s2c_recv = anyio.create_memory_object_stream(100)
+
+        from pydantic import TypeAdapter
+        from mcp.types import JSONRPCMessage
+        adapter = TypeAdapter(JSONRPCMessage)
+
+        init_done = anyio.Event()
+
+        async def srv_to_ws():
+            first = True 
+            try:
+                async for msg in s2c_recv:
+                    await ws.send_json(msg.model_dump())
+                    if first:
+                        init_done.set()
+                        first = False
+            finally:
+                # make sure cleanup survives TaskGroup cancellation
+                with anyio.CancelScope(shield=True):
+                    with suppress(RuntimeError):       # idempotent close
+                        await ws.close()
+
+        async def ws_to_srv():
+            try:
+                # 1st frame is always "initialize"
+                first = adapter.validate_python(await ws.receive_json())
+                await c2s_send.send(first)
+                await init_done.wait()          # block until server ready
+                while True:
+                    data = await ws.receive_json()
+                    await c2s_send.send(adapter.validate_python(data))
+            except WebSocketDisconnect:
+                await c2s_send.aclose()
+
+        async with anyio.create_task_group() as tg:
+            tg.start_soon(mcp.run, c2s_recv, s2c_send, init_opts)
+            tg.start_soon(ws_to_srv)
+            tg.start_soon(srv_to_ws)
+
+    # ── SSE transport (official) ─────────────────────────────
+    sse = SseServerTransport(f"{base}/messages/")
+
+    @app.get(f"{base}/sse")
+    async def _mcp_sse(request: Request):
+        async with sse.connect_sse(
+            request.scope, request.receive, request._send  # starlette ASGI primitives
+        ) as (read_stream, write_stream):
+            await mcp.run(read_stream, write_stream, init_opts)
+
+    # client → server frames are POSTed here
+    app.mount(f"{base}/messages", app=sse.handle_post_message)
+
+    # ── schema endpoint ───────────────────────────────────────
+    @app.get(f"{base}/schema")
+    async def _schema_endpoint():
+        return JSONResponse({
+            "tools": [x.model_dump() for x in await _list_tools()],
+            "resources": [x.model_dump() for x in await _list_resources()],
+            "resource_templates": [x.model_dump() for x in await _list_templates()],
+        })
+
+
+# ── helpers ────────────────────────────────────────────────────
+def _route_name(path: str) -> str:
+    return re.sub(r"[/{}}]", "_", path).strip("_")
+
+def _path_params(app: FastAPI, fn: Callable) -> List[str]:
+    for r in app.routes:
+        if r.endpoint is fn:
+            return list(r.param_convertors.keys())
+    return []

deploy/docker/requirements.txt

@@ -1,10 +1,17 @@
-crawl4ai
-fastapi
-uvicorn
+fastapi>=0.115.12
+uvicorn>=0.34.2
 gunicorn>=23.0.0
-slowapi>=0.1.9
-prometheus-fastapi-instrumentator>=7.0.2
+slowapi==0.1.9
+prometheus-fastapi-instrumentator>=7.1.0
 redis>=5.2.1
 jwt>=1.3.1
 dnspython>=2.7.0
-email-validator>=2.2.0
+email-validator==2.2.0
+sse-starlette==2.2.1
+pydantic>=2.11
+rank-bm25==0.2.2
+anyio==4.9.0
+PyJWT==2.10.1
+mcp>=1.6.0
+websockets>=15.0.1
+httpx[http2]>=0.27.2

deploy/docker/schemas.py

@@ -0,0 +1,42 @@
+from typing import List, Optional, Dict
+from enum import Enum
+from pydantic import BaseModel, Field
+from utils import FilterType
+
+
+class CrawlRequest(BaseModel):
+    urls: List[str] = Field(min_length=1, max_length=100)
+    browser_config: Optional[Dict] = Field(default_factory=dict)
+    crawler_config: Optional[Dict] = Field(default_factory=dict)
+
+class MarkdownRequest(BaseModel):
+    """Request body for the /md endpoint."""
+    url: str                    = Field(...,  description="Absolute http/https URL to fetch")
+    f:   FilterType             = Field(FilterType.FIT, description="Content‑filter strategy: fit, raw, bm25, or llm")
+    q:   Optional[str] = Field(None,  description="Query string used by BM25/LLM filters")
+    c:   Optional[str] = Field("0",   description="Cache‑bust / revision counter")
+    provider: Optional[str] = Field(None, description="LLM provider override (e.g., 'anthropic/claude-3-opus')")
+
+
+class RawCode(BaseModel):
+    code: str
+
+class HTMLRequest(BaseModel):
+    url: str
+    
+class ScreenshotRequest(BaseModel):
+    url: str
+    screenshot_wait_for: Optional[float] = 2
+    output_path: Optional[str] = None
+
+class PDFRequest(BaseModel):
+    url: str
+    output_path: Optional[str] = None
+
+
+class JSEndpointRequest(BaseModel):
+    url: str
+    scripts: List[str] = Field(
+        ...,
+        description="List of separated JavaScript snippets to execute"
+    )

deploy/docker/server.py

@@ -1,150 +1,449 @@
+# ───────────────────────── server.py ─────────────────────────
+"""
+Crawl4AI FastAPI entry‑point
+• Browser pool + global page cap
+• Rate‑limiting, security, metrics
+• /crawl, /crawl/stream, /md, /llm endpoints
+"""
+
+# ── stdlib & 3rd‑party imports ───────────────────────────────
+from crawler_pool import get_crawler, close_all, janitor
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from auth import create_access_token, get_token_dependency, TokenRequest
+from pydantic import BaseModel
+from typing import Optional, List, Dict
+from fastapi import Request, Depends
+from fastapi.responses import FileResponse
+import base64
+import re
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from api import (
+    handle_markdown_request, handle_llm_qa,
+    handle_stream_crawl_request, handle_crawl_request,
+    stream_results
+)
+from schemas import (
+    CrawlRequest,
+    MarkdownRequest,
+    RawCode,
+    HTMLRequest,
+    ScreenshotRequest,
+    PDFRequest,
+    JSEndpointRequest,
+)
+
+from utils import (
+    FilterType, load_config, setup_logging, verify_email_domain
+)
 import os
 import sys
 import time
-from typing import List, Optional, Dict
-from fastapi import FastAPI, HTTPException, Request, Query, Path, Depends
-from fastapi.responses import StreamingResponse, RedirectResponse, PlainTextResponse, JSONResponse
+import asyncio
+from typing import List
+from contextlib import asynccontextmanager
+import pathlib
+
+from fastapi import (
+    FastAPI, HTTPException, Request, Path, Query, Depends
+)
+from rank_bm25 import BM25Okapi
+from fastapi.responses import (
+    StreamingResponse, RedirectResponse, PlainTextResponse, JSONResponse
+)
 from fastapi.middleware.httpsredirect import HTTPSRedirectMiddleware
 from fastapi.middleware.trustedhost import TrustedHostMiddleware
+from fastapi.staticfiles import StaticFiles
+from job import init_job_router
+
+from mcp_bridge import attach_mcp, mcp_resource, mcp_template, mcp_tool
+
+import ast
+import crawl4ai as _c4
 from pydantic import BaseModel, Field
 from slowapi import Limiter
 from slowapi.util import get_remote_address
 from prometheus_fastapi_instrumentator import Instrumentator
 from redis import asyncio as aioredis
 
+# ── internal imports (after sys.path append) ─────────────────
 sys.path.append(os.path.dirname(os.path.realpath(__file__)))
-from utils import FilterType, load_config, setup_logging, verify_email_domain
-from api import (
-    handle_markdown_request,
-    handle_llm_qa,
-    handle_stream_crawl_request,
-    handle_crawl_request,
-    stream_results
-)
-from auth import create_access_token, get_token_dependency, TokenRequest  # Import from auth.py
 
-__version__ = "0.2.6"
-
-class CrawlRequest(BaseModel):
-    urls: List[str] = Field(min_length=1, max_length=100)
-    browser_config: Optional[Dict] = Field(default_factory=dict)
-    crawler_config: Optional[Dict] = Field(default_factory=dict)
-
-# Load configuration and setup
+# ────────────────── configuration / logging ──────────────────
 config = load_config()
 setup_logging(config)
 
-# Initialize Redis
+__version__ = "0.5.1-d1"
+
+# ── global page semaphore (hard cap) ─────────────────────────
+MAX_PAGES = config["crawler"]["pool"].get("max_pages", 30)
+GLOBAL_SEM = asyncio.Semaphore(MAX_PAGES)
+
+# import logging
+# page_log = logging.getLogger("page_cap")
+# orig_arun = AsyncWebCrawler.arun
+# async def capped_arun(self, *a, **kw):
+#     await GLOBAL_SEM.acquire()                        # ← take slot
+#     try:
+#         in_flight = MAX_PAGES - GLOBAL_SEM._value     # used permits
+#         page_log.info("🕸️  pages_in_flight=%s / %s", in_flight, MAX_PAGES)
+#         return await orig_arun(self, *a, **kw)
+#     finally:
+#         GLOBAL_SEM.release()                          # ← free slot
+
+orig_arun = AsyncWebCrawler.arun
+
+
+async def capped_arun(self, *a, **kw):
+    async with GLOBAL_SEM:
+        return await orig_arun(self, *a, **kw)
+AsyncWebCrawler.arun = capped_arun
+
+# ───────────────────── FastAPI lifespan ──────────────────────
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    await get_crawler(BrowserConfig(
+        extra_args=config["crawler"]["browser"].get("extra_args", []),
+        **config["crawler"]["browser"].get("kwargs", {}),
+    ))           # warm‑up
+    app.state.janitor = asyncio.create_task(janitor())        # idle GC
+    yield
+    app.state.janitor.cancel()
+    await close_all()
+
+# ───────────────────── FastAPI instance ──────────────────────
+app = FastAPI(
+    title=config["app"]["title"],
+    version=config["app"]["version"],
+    lifespan=lifespan,
+)
+
+# ── static playground ──────────────────────────────────────
+STATIC_DIR = pathlib.Path(__file__).parent / "static" / "playground"
+if not STATIC_DIR.exists():
+    raise RuntimeError(f"Playground assets not found at {STATIC_DIR}")
+app.mount(
+    "/playground",
+    StaticFiles(directory=STATIC_DIR, html=True),
+    name="play",
+)
+
+
+@app.get("/")
+async def root():
+    return RedirectResponse("/playground")
+
+# ─────────────────── infra / middleware  ─────────────────────
 redis = aioredis.from_url(config["redis"].get("uri", "redis://localhost"))
 
-# Initialize rate limiter
 limiter = Limiter(
     key_func=get_remote_address,
     default_limits=[config["rate_limiting"]["default_limit"]],
-    storage_uri=config["rate_limiting"]["storage_uri"]
+    storage_uri=config["rate_limiting"]["storage_uri"],
 )
 
-app = FastAPI(
-    title=config["app"]["title"],
-    version=config["app"]["version"]
-)
 
-# Configure middleware
-def setup_security_middleware(app, config):
-    sec_config = config.get("security", {})
-    if sec_config.get("enabled", False):
-        if sec_config.get("https_redirect", False):
-            app.add_middleware(HTTPSRedirectMiddleware)
-        if sec_config.get("trusted_hosts", []) != ["*"]:
-            app.add_middleware(TrustedHostMiddleware, allowed_hosts=sec_config["trusted_hosts"])
+def _setup_security(app_: FastAPI):
+    sec = config["security"]
+    if not sec["enabled"]:
+        return
+    if sec.get("https_redirect"):
+        app_.add_middleware(HTTPSRedirectMiddleware)
+    if sec.get("trusted_hosts", []) != ["*"]:
+        app_.add_middleware(
+            TrustedHostMiddleware, allowed_hosts=sec["trusted_hosts"]
+        )
 
-setup_security_middleware(app, config)
 
-# Prometheus instrumentation
+_setup_security(app)
+
 if config["observability"]["prometheus"]["enabled"]:
     Instrumentator().instrument(app).expose(app)
 
-# Get token dependency based on config
-token_dependency = get_token_dependency(config)
+token_dep = get_token_dependency(config)
+
 
-# Middleware for security headers
 @app.middleware("http")
 async def add_security_headers(request: Request, call_next):
-    response = await call_next(request)
+    resp = await call_next(request)
     if config["security"]["enabled"]:
-        response.headers.update(config["security"]["headers"])
-    return response
+        resp.headers.update(config["security"]["headers"])
+    return resp
 
-# Token endpoint (always available, but usage depends on config)
+# ───────────────── safe config‑dump helper ─────────────────
+ALLOWED_TYPES = {
+    "CrawlerRunConfig": CrawlerRunConfig,
+    "BrowserConfig": BrowserConfig,
+}
+
+
+def _safe_eval_config(expr: str) -> dict:
+    """
+    Accept exactly one top‑level call to CrawlerRunConfig(...) or BrowserConfig(...).
+    Whatever is inside the parentheses is fine *except* further function calls
+    (so no  __import__('os') stuff).  All public names from crawl4ai are available
+    when we eval.
+    """
+    tree = ast.parse(expr, mode="eval")
+
+    # must be a single call
+    if not isinstance(tree.body, ast.Call):
+        raise ValueError("Expression must be a single constructor call")
+
+    call = tree.body
+    if not (isinstance(call.func, ast.Name) and call.func.id in {"CrawlerRunConfig", "BrowserConfig"}):
+        raise ValueError(
+            "Only CrawlerRunConfig(...) or BrowserConfig(...) are allowed")
+
+    # forbid nested calls to keep the surface tiny
+    for node in ast.walk(call):
+        if isinstance(node, ast.Call) and node is not call:
+            raise ValueError("Nested function calls are not permitted")
+
+    # expose everything that crawl4ai exports, nothing else
+    safe_env = {name: getattr(_c4, name)
+                for name in dir(_c4) if not name.startswith("_")}
+    obj = eval(compile(tree, "<config>", "eval"),
+               {"__builtins__": {}}, safe_env)
+    return obj.dump()
+
+
+# ── job router ──────────────────────────────────────────────
+app.include_router(init_job_router(redis, config, token_dep))
+
+# ──────────────────────── Endpoints ──────────────────────────
 @app.post("/token")
-async def get_token(request_data: TokenRequest):
-    if not verify_email_domain(request_data.email):
-        raise HTTPException(status_code=400, detail="Invalid email domain")
-    token = create_access_token({"sub": request_data.email})
-    return {"email": request_data.email, "access_token": token, "token_type": "bearer"}
+async def get_token(req: TokenRequest):
+    if not verify_email_domain(req.email):
+        raise HTTPException(400, "Invalid email domain")
+    token = create_access_token({"sub": req.email})
+    return {"email": req.email, "access_token": token, "token_type": "bearer"}
 
-# Endpoints with conditional auth
-@app.get("/md/{url:path}")
+
+@app.post("/config/dump")
+async def config_dump(raw: RawCode):
+    try:
+        return JSONResponse(_safe_eval_config(raw.code.strip()))
+    except Exception as e:
+        raise HTTPException(400, str(e))
+
+
+@app.post("/md")
 @limiter.limit(config["rate_limiting"]["default_limit"])
+@mcp_tool("md")
 async def get_markdown(
     request: Request,
-    url: str,
-    f: FilterType = FilterType.FIT,
-    q: Optional[str] = None,
-    c: Optional[str] = "0",
-    token_data: Optional[Dict] = Depends(token_dependency)
+    body: MarkdownRequest,
+    _td: Dict = Depends(token_dep),
 ):
-    result = await handle_markdown_request(url, f, q, c, config)
-    return PlainTextResponse(result)
+    if not body.url.startswith(("http://", "https://")):
+        raise HTTPException(
+            400, "URL must be absolute and start with http/https")
+    markdown = await handle_markdown_request(
+        body.url, body.f, body.q, body.c, config, body.provider
+    )
+    return JSONResponse({
+        "url": body.url,
+        "filter": body.f,
+        "query": body.q,
+        "cache": body.c,
+        "markdown": markdown,
+        "success": True
+    })
 
-@app.get("/llm/{url:path}", description="URL should be without http/https prefix")
+
+@app.post("/html")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+@mcp_tool("html")
+async def generate_html(
+    request: Request,
+    body: HTMLRequest,
+    _td: Dict = Depends(token_dep),
+):
+    """
+    Crawls the URL, preprocesses the raw HTML for schema extraction, and returns the processed HTML.
+    Use when you need sanitized HTML structures for building schemas or further processing.
+    """
+    cfg = CrawlerRunConfig()
+    async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
+        results = await crawler.arun(url=body.url, config=cfg)
+    raw_html = results[0].html
+    from crawl4ai.utils import preprocess_html_for_schema
+    processed_html = preprocess_html_for_schema(raw_html)
+    return JSONResponse({"html": processed_html, "url": body.url, "success": True})
+
+# Screenshot endpoint
+
+
+@app.post("/screenshot")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+@mcp_tool("screenshot")
+async def generate_screenshot(
+    request: Request,
+    body: ScreenshotRequest,
+    _td: Dict = Depends(token_dep),
+):
+    """
+    Capture a full-page PNG screenshot of the specified URL, waiting an optional delay before capture,
+    Use when you need an image snapshot of the rendered page. Its recommened to provide an output path to save the screenshot.
+    Then in result instead of the screenshot you will get a path to the saved file.
+    """
+    cfg = CrawlerRunConfig(
+        screenshot=True, screenshot_wait_for=body.screenshot_wait_for)
+    async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
+        results = await crawler.arun(url=body.url, config=cfg)
+    screenshot_data = results[0].screenshot
+    if body.output_path:
+        abs_path = os.path.abspath(body.output_path)
+        os.makedirs(os.path.dirname(abs_path), exist_ok=True)
+        with open(abs_path, "wb") as f:
+            f.write(base64.b64decode(screenshot_data))
+        return {"success": True, "path": abs_path}
+    return {"success": True, "screenshot": screenshot_data}
+
+# PDF endpoint
+
+
+@app.post("/pdf")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+@mcp_tool("pdf")
+async def generate_pdf(
+    request: Request,
+    body: PDFRequest,
+    _td: Dict = Depends(token_dep),
+):
+    """
+    Generate a PDF document of the specified URL,
+    Use when you need a printable or archivable snapshot of the page. It is recommended to provide an output path to save the PDF.
+    Then in result instead of the PDF you will get a path to the saved file.
+    """
+    cfg = CrawlerRunConfig(pdf=True)
+    async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
+        results = await crawler.arun(url=body.url, config=cfg)
+    pdf_data = results[0].pdf
+    if body.output_path:
+        abs_path = os.path.abspath(body.output_path)
+        os.makedirs(os.path.dirname(abs_path), exist_ok=True)
+        with open(abs_path, "wb") as f:
+            f.write(pdf_data)
+        return {"success": True, "path": abs_path}
+    return {"success": True, "pdf": base64.b64encode(pdf_data).decode()}
+
+
+@app.post("/execute_js")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+@mcp_tool("execute_js")
+async def execute_js(
+    request: Request,
+    body: JSEndpointRequest,
+    _td: Dict = Depends(token_dep),
+):
+    """
+    Execute a sequence of JavaScript snippets on the specified URL.
+    Return the full CrawlResult JSON (first result).
+    Use this when you need to interact with dynamic pages using JS.
+    REMEMBER: Scripts accept a list of separated JS snippets to execute and execute them in order.
+    IMPORTANT: Each script should be an expression that returns a value. It can be an IIFE or an async function. You can think of it as such.
+        Your script will replace '{script}' and execute in the browser context. So provide either an IIFE or a sync/async function that returns a value.
+    Return Format:
+        - The return result is an instance of CrawlResult, so you have access to markdown, links, and other stuff. If this is enough, you don't need to call again for other endpoints.
+
+        ```python
+        class CrawlResult(BaseModel):
+            url: str
+            html: str
+            success: bool
+            cleaned_html: Optional[str] = None
+            media: Dict[str, List[Dict]] = {}
+            links: Dict[str, List[Dict]] = {}
+            downloaded_files: Optional[List[str]] = None
+            js_execution_result: Optional[Dict[str, Any]] = None
+            screenshot: Optional[str] = None
+            pdf: Optional[bytes] = None
+            mhtml: Optional[str] = None
+            _markdown: Optional[MarkdownGenerationResult] = PrivateAttr(default=None)
+            extracted_content: Optional[str] = None
+            metadata: Optional[dict] = None
+            error_message: Optional[str] = None
+            session_id: Optional[str] = None
+            response_headers: Optional[dict] = None
+            status_code: Optional[int] = None
+            ssl_certificate: Optional[SSLCertificate] = None
+            dispatch_result: Optional[DispatchResult] = None
+            redirected_url: Optional[str] = None
+            network_requests: Optional[List[Dict[str, Any]]] = None
+            console_messages: Optional[List[Dict[str, Any]]] = None
+
+        class MarkdownGenerationResult(BaseModel):
+            raw_markdown: str
+            markdown_with_citations: str
+            references_markdown: str
+            fit_markdown: Optional[str] = None
+            fit_html: Optional[str] = None
+        ```
+
+    """
+    cfg = CrawlerRunConfig(js_code=body.scripts)
+    async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
+        results = await crawler.arun(url=body.url, config=cfg)
+    # Return JSON-serializable dict of the first CrawlResult
+    data = results[0].model_dump()
+    return JSONResponse(data)
+
+
+@app.get("/llm/{url:path}")
 async def llm_endpoint(
     request: Request,
     url: str = Path(...),
-    q: Optional[str] = Query(None),
-    token_data: Optional[Dict] = Depends(token_dependency)
+    q: str = Query(...),
+    _td: Dict = Depends(token_dep),
 ):
     if not q:
-        raise HTTPException(status_code=400, detail="Query parameter 'q' is required")
-    if not url.startswith(('http://', 'https://')):
-        url = 'https://' + url
-    try:
-        answer = await handle_llm_qa(url, q, config)
-        return JSONResponse({"answer": answer})
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+        raise HTTPException(400, "Query parameter 'q' is required")
+    if not url.startswith(("http://", "https://")):
+        url = "https://" + url
+    answer = await handle_llm_qa(url, q, config)
+    return JSONResponse({"answer": answer})
+
 
 @app.get("/schema")
 async def get_schema():
     from crawl4ai import BrowserConfig, CrawlerRunConfig
-    return {"browser": BrowserConfig().dump(), "crawler": CrawlerRunConfig().dump()}
+    return {"browser": BrowserConfig().dump(),
+            "crawler": CrawlerRunConfig().dump()}
+
 
 @app.get(config["observability"]["health_check"]["endpoint"])
 async def health():
     return {"status": "ok", "timestamp": time.time(), "version": __version__}
 
+
 @app.get(config["observability"]["prometheus"]["endpoint"])
 async def metrics():
-    return RedirectResponse(url=config["observability"]["prometheus"]["endpoint"])
+    return RedirectResponse(config["observability"]["prometheus"]["endpoint"])
+
 
 @app.post("/crawl")
 @limiter.limit(config["rate_limiting"]["default_limit"])
+@mcp_tool("crawl")
 async def crawl(
     request: Request,
     crawl_request: CrawlRequest,
-    token_data: Optional[Dict] = Depends(token_dependency)
+    _td: Dict = Depends(token_dep),
 ):
+    """
+    Crawl a list of URLs and return the results as JSON.
+    """
     if not crawl_request.urls:
-        raise HTTPException(status_code=400, detail="At least one URL required")
-    
-    results = await handle_crawl_request(
+        raise HTTPException(400, "At least one URL required")
+    res = await handle_crawl_request(
         urls=crawl_request.urls,
         browser_config=crawl_request.browser_config,
         crawler_config=crawl_request.crawler_config,
-        config=config
+        config=config,
     )
-
-    return JSONResponse(results)
+    return JSONResponse(res)
 
 
 @app.post("/crawl/stream")
@@ -152,24 +451,161 @@ async def crawl(
 async def crawl_stream(
     request: Request,
     crawl_request: CrawlRequest,
-    token_data: Optional[Dict] = Depends(token_dependency)
+    _td: Dict = Depends(token_dep),
 ):
     if not crawl_request.urls:
-        raise HTTPException(status_code=400, detail="At least one URL required")
-
-    crawler, results_gen = await handle_stream_crawl_request(
+        raise HTTPException(400, "At least one URL required")
+    crawler, gen = await handle_stream_crawl_request(
         urls=crawl_request.urls,
         browser_config=crawl_request.browser_config,
         crawler_config=crawl_request.crawler_config,
-        config=config
+        config=config,
     )
-
     return StreamingResponse(
-        stream_results(crawler, results_gen),
-        media_type='application/x-ndjson',
-        headers={'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'X-Stream-Status': 'active'}
+        stream_results(crawler, gen),
+        media_type="application/x-ndjson",
+        headers={
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Stream-Status": "active",
+        },
     )
 
+
+def chunk_code_functions(code_md: str) -> List[str]:
+    """Extract each function/class from markdown code blocks per file."""
+    pattern = re.compile(
+        # match "## File: <path>" then a ```py fence, then capture until the closing ```
+        r'##\s*File:\s*(?P<path>.+?)\s*?\r?\n'      # file header
+        r'```py\s*?\r?\n'                         # opening fence
+        r'(?P<code>.*?)(?=\r?\n```)',             # code block
+        re.DOTALL
+    )
+    chunks: List[str] = []
+    for m in pattern.finditer(code_md):
+        file_path = m.group("path").strip()
+        code_blk = m.group("code")
+        tree = ast.parse(code_blk)
+        lines = code_blk.splitlines()
+        for node in tree.body:
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                start = node.lineno - 1
+                end = getattr(node, "end_lineno", start + 1)
+                snippet = "\n".join(lines[start:end])
+                chunks.append(f"# File: {file_path}\n{snippet}")
+    return chunks
+
+
+def chunk_doc_sections(doc: str) -> List[str]:
+    lines = doc.splitlines(keepends=True)
+    sections = []
+    current: List[str] = []
+    for line in lines:
+        if re.match(r"^#{1,6}\s", line):
+            if current:
+                sections.append("".join(current))
+            current = [line]
+        else:
+            current.append(line)
+    if current:
+        sections.append("".join(current))
+    return sections
+
+
+@app.get("/ask")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+@mcp_tool("ask")
+async def get_context(
+    request: Request,
+    _td: Dict = Depends(token_dep),
+    context_type: str = Query("all", regex="^(code|doc|all)$"),
+    query: Optional[str] = Query(
+        None, description="search query to filter chunks"),
+    score_ratio: float = Query(
+        0.5, ge=0.0, le=1.0, description="min score as fraction of max_score"),
+    max_results: int = Query(
+        20, ge=1, description="absolute cap on returned chunks"),
+):
+    """
+    This end point is design for any questions about Crawl4ai library. It returns a plain text markdown with extensive information about Crawl4ai. 
+    You can use this as a context for any AI assistant. Use this endpoint for AI assistants to retrieve library context for decision making or code generation tasks.
+    Alway is BEST practice you provide a query to filter the context. Otherwise the lenght of the response will be very long.
+
+    Parameters:
+    - context_type: Specify "code" for code context, "doc" for documentation context, or "all" for both.
+    - query: RECOMMENDED search query to filter paragraphs using BM25. You can leave this empty to get all the context.
+    - score_ratio: Minimum score as a fraction of the maximum score for filtering results.
+    - max_results: Maximum number of results to return. Default is 20.
+
+    Returns:
+    - JSON response with the requested context.
+    - If "code" is specified, returns the code context.
+    - If "doc" is specified, returns the documentation context.
+    - If "all" is specified, returns both code and documentation contexts.
+    """
+    # load contexts
+    base = os.path.dirname(__file__)
+    code_path = os.path.join(base, "c4ai-code-context.md")
+    doc_path = os.path.join(base, "c4ai-doc-context.md")
+    if not os.path.exists(code_path) or not os.path.exists(doc_path):
+        raise HTTPException(404, "Context files not found")
+
+    with open(code_path, "r") as f:
+        code_content = f.read()
+    with open(doc_path, "r") as f:
+        doc_content = f.read()
+
+    # if no query, just return raw contexts
+    if not query:
+        if context_type == "code":
+            return JSONResponse({"code_context": code_content})
+        if context_type == "doc":
+            return JSONResponse({"doc_context": doc_content})
+        return JSONResponse({
+            "code_context": code_content,
+            "doc_context": doc_content,
+        })
+
+    tokens = query.split()
+    results: Dict[str, List[Dict[str, float]]] = {}
+
+    # code BM25 over functions/classes
+    if context_type in ("code", "all"):
+        code_chunks = chunk_code_functions(code_content)
+        bm25 = BM25Okapi([c.split() for c in code_chunks])
+        scores = bm25.get_scores(tokens)
+        max_sc = float(scores.max()) if scores.size > 0 else 0.0
+        cutoff = max_sc * score_ratio
+        picked = [(c, s) for c, s in zip(code_chunks, scores) if s >= cutoff]
+        picked = sorted(picked, key=lambda x: x[1], reverse=True)[:max_results]
+        results["code_results"] = [{"text": c, "score": s} for c, s in picked]
+
+    # doc BM25 over markdown sections
+    if context_type in ("doc", "all"):
+        sections = chunk_doc_sections(doc_content)
+        bm25d = BM25Okapi([sec.split() for sec in sections])
+        scores_d = bm25d.get_scores(tokens)
+        max_sd = float(scores_d.max()) if scores_d.size > 0 else 0.0
+        cutoff_d = max_sd * score_ratio
+        idxs = [i for i, s in enumerate(scores_d) if s >= cutoff_d]
+        neighbors = set(i for idx in idxs for i in (idx-1, idx, idx+1))
+        valid = [i for i in sorted(neighbors) if 0 <= i < len(sections)]
+        valid = valid[:max_results]
+        results["doc_results"] = [
+            {"text": sections[i], "score": scores_d[i]} for i in valid
+        ]
+
+    return JSONResponse(results)
+
+
+# attach MCP layer (adds /mcp/ws, /mcp/sse, /mcp/schema)
+print(f"MCP server running on {config['app']['host']}:{config['app']['port']}")
+attach_mcp(
+    app,
+    base_url=f"http://{config['app']['host']}:{config['app']['port']}"
+)
+
+# ────────────────────────── cli ──────────────────────────────
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(
@@ -177,5 +613,6 @@ if __name__ == "__main__":
         host=config["app"]["host"],
         port=config["app"]["port"],
         reload=config["app"]["reload"],
-        timeout_keep_alive=config["app"]["timeout_keep_alive"]
-    )
+        timeout_keep_alive=config["app"]["timeout_keep_alive"],
+    )
+# ─────────────────────────────────────────────────────────────

deploy/docker/static/playground/index.html

@@ -0,0 +1,965 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Crawl4AI Playground</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+    <script>
+        tailwind.config = {
+            theme: {
+                extend: {
+                    colors: {
+                        primary: '#4EFFFF',
+                        primarydim: '#09b5a5',
+                        accent: '#F380F5',
+                        dark: '#070708',
+                        light: '#E8E9ED',
+                        secondary: '#D5CEBF',
+                        codebg: '#1E1E1E',
+                        surface: '#202020',
+                        border: '#3F3F44',
+                    },
+                    fontFamily: {
+                        mono: ['Fira Code', 'monospace'],
+                    },
+                }
+            }
+        }
+    </script>
+    <link href="https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;500&display=swap" rel="stylesheet">
+    <!-- Highlight.js -->
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.11/clipboard.min.js"></script>
+    <!-- CodeMirror (python mode) -->
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/codemirror.min.css">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/codemirror.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/mode/python/python.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/addon/edit/matchbrackets.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/addon/selection/active-line.min.js"></script>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/5.65.16/theme/darcula.min.css">
+    <!-- <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/python.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/bash.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/json.min.js"></script> -->
+    <style>
+        /* Custom CodeMirror styling to match theme */
+        .CodeMirror {
+            background-color: #1E1E1E !important;
+            color: #E8E9ED !important;
+            border-radius: 4px;
+            font-family: 'Fira Code', monospace;
+            font-size: 0.9rem;
+        }
+        
+        .CodeMirror-gutters {
+            background-color: #1E1E1E !important;
+            border-right: 1px solid #3F3F44 !important;
+        }
+        
+        .CodeMirror-linenumber {
+            color: #3F3F44 !important;
+        }
+        
+        .cm-s-darcula .cm-keyword {
+            color: #4EFFFF !important;
+        }
+        
+        .cm-s-darcula .cm-string {
+            color: #F380F5 !important;
+        }
+        
+        .cm-s-darcula .cm-number {
+            color: #D5CEBF !important;
+        }
+        
+        /* Add to your <style> section or Tailwind config */
+        .hljs {
+            background: #1E1E1E !important;
+            border-radius: 4px;
+            padding: 1rem !important;
+        }
+
+        pre code.hljs {
+            display: block;
+            overflow-x: auto;
+        }
+
+        /* Language-specific colors */
+        .hljs-attr {
+            color: #4EFFFF;
+        }
+
+        /* JSON keys */
+        .hljs-string {
+            color: #F380F5;
+        }
+
+        /* Strings */
+        .hljs-number {
+            color: #D5CEBF;
+        }
+
+        /* Numbers */
+        .hljs-keyword {
+            color: #4EFFFF;
+        }
+
+        pre code {
+            white-space: pre-wrap;
+            word-break: break-word;
+        }
+
+        .copy-btn {
+            transition: all 0.2s ease;
+            opacity: 0.7;
+        }
+
+        .copy-btn:hover {
+            opacity: 1;
+        }
+
+        .tab-content:hover .copy-btn {
+            opacity: 0.7;
+        }
+
+        .tab-content:hover .copy-btn:hover {
+            opacity: 1;
+        }
+
+        /* copid text highlighted */
+        .highlighted {
+            background-color: rgba(78, 255, 255, 0.2) !important;
+            transition: background-color 0.5s ease;
+        }
+    </style>
+</head>
+
+<body class="bg-dark text-light font-mono min-h-screen flex flex-col" style="font-feature-settings: 'calt' 0;">
+    <!-- Header -->
+    <header class="border-b border-border px-4 py-2 flex items-center">
+        <h1 class="text-lg font-medium flex items-center space-x-4">
+            <span>🚀🤖 <span class="text-primary">Crawl4AI</span> Playground</span>
+
+            <!-- GitHub badges -->
+            <a href="https://github.com/unclecode/crawl4ai" target="_blank" class="flex space-x-1">
+                <img src="https://img.shields.io/github/stars/unclecode/crawl4ai?style=social"
+                     alt="GitHub stars" class="h-5">
+                <img src="https://img.shields.io/github/forks/unclecode/crawl4ai?style=social"
+                     alt="GitHub forks" class="h-5">
+            </a>
+
+            <!-- Docs -->
+            <a href="https://docs.crawl4ai.com" target="_blank"
+               class="text-xs text-secondary hover:text-primary underline flex items-center">
+                Docs
+            </a>
+
+            <!-- X (Twitter) follow -->
+            <a href="https://x.com/unclecode" target="_blank"
+               class="hover:text-primary flex items-center" title="Follow @unclecode on X">
+                <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"
+                     class="w-4 h-4 fill-current mr-1">
+                    <path d="M22.46 6c-.77.35-1.6.58-2.46.69a4.27 4.27 0 001.88-2.35 8.53 8.53 0 01-2.71 1.04 4.24 4.24 0 00-7.23 3.87A12.05 12.05 0 013 4.62a4.24 4.24 0 001.31 5.65 4.2 4.2 0 01-1.92-.53v.05a4.24 4.24 0 003.4 4.16 4.31 4.31 0 01-1.91.07 4.25 4.25 0 003.96 2.95A8.5 8.5 0 012 19.55a12.04 12.04 0 006.53 1.92c7.84 0 12.13-6.49 12.13-12.13 0-.18-.01-.36-.02-.54A8.63 8.63 0 0024 5.1a8.45 8.45 0 01-2.54.7z"/>
+                </svg>
+                <span class="text-xs">@unclecode</span>
+            </a>
+        </h1>
+
+        <div class="ml-auto flex space-x-2">
+            <button id="play-tab"
+                class="px-3 py-1 rounded-t bg-surface border border-b-0 border-border text-primary">Playground</button>
+            <button id="stress-tab" class="px-3 py-1 rounded-t border border-border hover:bg-surface">Stress
+                Test</button>
+        </div>
+    </header>
+
+    <!-- Main Playground -->
+    <main id="playground" class="flex-1 flex flex-col p-4 space-y-4 max-w-5xl w-full mx-auto">
+        <!-- Request Builder -->
+        <section class="bg-surface rounded-lg border border-border overflow-hidden">
+            <div class="px-4 py-2 border-b border-border flex items-center">
+                <h2 class="font-medium">Request Builder</h2>
+                <select id="endpoint" class="ml-auto bg-dark border border-border rounded px-2 py-1 text-sm">
+                    <option value="crawl">/crawl (batch)</option>
+                    <option value="crawl_stream">/crawl/stream</option>
+                    <option value="md">/md</option>
+                    <option value="llm">/llm</option>
+                </select>
+            </div>
+            <div class="p-4">
+                <label class="block mb-2 text-sm">URL(s) - one per line</label>
+                <textarea id="urls" class="w-full bg-dark border border-border rounded p-2 h-32 text-sm mb-4"
+                    spellcheck="false">https://example.com</textarea>
+
+                <!-- Specific options for /md endpoint -->
+                <details id="md-options" class="mb-4 hidden">
+                    <summary class="text-sm text-secondary cursor-pointer">/md Options</summary>
+                    <div class="mt-2 space-y-3 p-2 border border-border rounded">
+                        <div>
+                            <label for="md-filter" class="block text-xs text-secondary mb-1">Filter Type</label>
+                            <select id="md-filter" class="bg-dark border border-border rounded px-2 py-1 text-sm w-full">
+                                <option value="fit">fit - Adaptive content filtering</option>
+                                <option value="raw">raw - No filtering</option>
+                                <option value="bm25">bm25 - BM25 keyword relevance</option>
+                                <option value="llm">llm - LLM-based filtering</option>
+                            </select>
+                        </div>
+                        <div>
+                            <label for="md-query" class="block text-xs text-secondary mb-1">Query (for BM25/LLM filters)</label>
+                            <input id="md-query" type="text" placeholder="Enter search terms or instructions" 
+                                class="bg-dark border border-border rounded px-2 py-1 text-sm w-full">
+                        </div>
+                        <div>
+                            <label for="md-cache" class="block text-xs text-secondary mb-1">Cache Mode</label>
+                            <select id="md-cache" class="bg-dark border border-border rounded px-2 py-1 text-sm w-full">
+                                <option value="0">Write-Only (0)</option>
+                                <option value="1">Enabled (1)</option>
+                            </select>
+                        </div>
+                    </div>
+                </details>
+
+                <!-- Specific options for /llm endpoint -->
+                <details id="llm-options" class="mb-4 hidden">
+                    <summary class="text-sm text-secondary cursor-pointer">/llm Options</summary>
+                    <div class="mt-2 space-y-3 p-2 border border-border rounded">
+                        <div>
+                            <label for="llm-question" class="block text-xs text-secondary mb-1">Question</label>
+                            <input id="llm-question" type="text" value="What is this page about?" 
+                                class="bg-dark border border-border rounded px-2 py-1 text-sm w-full">
+                        </div>
+                    </div>
+                </details>
+
+                <!-- Advanced config for /crawl endpoints -->
+                <details id="adv-config" class="mb-4">
+                    <summary class="text-sm text-secondary cursor-pointer">Advanced Config <span
+                        class="text-xs text-primary">(Python → auto‑JSON)</span></summary>
+
+                    <!-- Toolbar -->
+                    <div class="flex items-center justify-end space-x-3 mt-2">
+                        <label for="cfg-type" class="text-xs text-secondary">Type:</label>
+                        <select id="cfg-type"
+                                class="bg-dark border border-border rounded px-1 py-0.5 text-xs">
+                            <option value="CrawlerRunConfig">CrawlerRunConfig</option>
+                            <option value="BrowserConfig">BrowserConfig</option>
+                        </select>
+
+                        <!-- help link -->
+                        <a href="https://docs.crawl4ai.com/api/parameters/"
+                           target="_blank"
+                           class="text-xs text-primary hover:underline flex items-center space-x-1"
+                           title="Open parameter reference in new tab">
+                            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"
+                                 class="w-4 h-4 fill-current">
+                                 <path d="M13 3h8v8h-2V6.41l-9.29 9.3-1.42-1.42 9.3-9.29H13V3z"/>
+                                 <path d="M5 5h4V3H3v6h2V5zm0 14v-4H3v6h6v-2H5z"/>
+                            </svg>
+                            <span>Docs</span>
+                        </a>
+
+                        <span id="cfg-status" class="text-xs text-secondary ml-2"></span>
+                    </div>
+
+                    <!-- CodeMirror host -->
+                    <div id="adv-editor" class="mt-2 border border-border rounded overflow-hidden h-40"></div>
+                </details>
+
+                <div class="flex space-x-2">
+                    <button id="run-btn" class="bg-primary text-dark px-4 py-2 rounded hover:bg-primarydim font-medium">
+                        Run (⌘/Ctrl+Enter)
+                    </button>
+                    <button id="export-btn" class="border border-border px-4 py-2 rounded hover:bg-surface hidden">
+                        Export Python Code
+                    </button>
+                </div>
+            </div>
+        </section>
+
+        <!-- Execution Status -->
+        <section id="execution-status" class="hidden bg-surface rounded-lg border border-border p-3 text-sm">
+            <div class="flex space-x-4">
+                <div id="status-badge" class="flex items-center">
+                    <span class="w-3 h-3 rounded-full mr-2"></span>
+                    <span>Ready</span>
+                </div>
+                <div>
+                    <span class="text-secondary">Time:</span>
+                    <span id="exec-time" class="text-light">-</span>
+                </div>
+                <div>
+                    <span class="text-secondary">Memory:</span>
+                    <span id="exec-mem" class="text-light">-</span>
+                </div>
+            </div>
+        </section>
+
+        <!-- Response Viewer -->
+        <!-- Update the Response Viewer section -->
+        <section class="bg-surface rounded-lg border border-border overflow-hidden flex-1 flex flex-col">
+            <div class="border-b border-border flex">
+                <button data-tab="response" class="tab-btn active px-4 py-2 border-r border-border">Response</button>
+                <button data-tab="python" class="tab-btn px-4 py-2 border-r border-border">Python</button>
+                <button data-tab="curl" class="tab-btn px-4 py-2">cURL</button>
+            </div>
+            <div class="flex-1 overflow-auto relative">
+                <!-- Response Tab -->
+                <div class="tab-content active h-full">
+                    <div class="absolute right-2 top-2">
+                        <button class="copy-btn bg-surface border border-border rounded px-2 py-1 text-xs hover:bg-dark"
+                            data-target="#response-content code">
+                            Copy
+                        </button>
+                    </div>
+                    <pre id="response-content" class="p-4 text-sm h-full"><code class="json hljs">{}</code></pre>
+                </div>
+
+                <!-- Python Tab -->
+                <div class="tab-content hidden h-full">
+                    <div class="absolute right-2 top-2">
+                        <button class="copy-btn bg-surface border border-border rounded px-2 py-1 text-xs hover:bg-dark"
+                            data-target="#python-content code">
+                            Copy
+                        </button>
+                    </div>
+                    <pre id="python-content" class="p-4 text-sm h-full"><code class="python hljs"></code></pre>
+                </div>
+
+                <!-- cURL Tab -->
+                <div class="tab-content hidden h-full">
+                    <div class="absolute right-2 top-2">
+                        <button class="copy-btn bg-surface border border-border rounded px-2 py-1 text-xs hover:bg-dark"
+                            data-target="#curl-content code">
+                            Copy
+                        </button>
+                    </div>
+                    <pre id="curl-content" class="p-4 text-sm h-full"><code class="bash hljs"></code></pre>
+                </div>
+            </div>
+        </section>
+    </main>
+
+    <!-- Stress Test Modal -->
+    <div id="stress-modal"
+        class="hidden fixed inset-0 bg-black bg-opacity-70 z-50 flex items-center justify-center p-4">
+        <div class="bg-surface rounded-lg border border-accent w-full max-w-3xl max-h-[90vh] flex flex-col">
+            <div class="px-4 py-2 border-b border-border flex items-center">
+                <h2 class="font-medium text-accent">🔥 Stress Test</h2>
+                <button id="close-stress" class="ml-auto text-secondary hover:text-light">&times;</button>
+            </div>
+
+            <div class="p-4 space-y-4 flex-1 overflow-auto">
+                <div class="grid grid-cols-3 gap-4">
+                    <div>
+                        <label class="block text-sm mb-1">Total URLs</label>
+                        <input id="st-total" type="number" value="20"
+                            class="w-full bg-dark border border-border rounded px-3 py-1">
+                    </div>
+                    <div>
+                        <label class="block text-sm mb-1">Chunk Size</label>
+                        <input id="st-chunk" type="number" value="5"
+                            class="w-full bg-dark border border-border rounded px-3 py-1">
+                    </div>
+                    <div>
+                        <label class="block text-sm mb-1">Concurrency</label>
+                        <input id="st-conc" type="number" value="2"
+                            class="w-full bg-dark border border-border rounded px-3 py-1">
+                    </div>
+                </div>
+
+                <div class="flex items-center">
+                    <input id="st-stream" type="checkbox" class="mr-2">
+                    <label for="st-stream" class="text-sm">Use /crawl/stream</label>
+                    <button id="st-run"
+                        class="ml-auto bg-accent text-dark px-4 py-2 rounded hover:bg-opacity-90 font-medium">
+                        Run Stress Test
+                    </button>
+                </div>
+
+                <div class="mt-4">
+                    <div class="bg-dark rounded border border-border p-3 h-64 overflow-auto text-sm whitespace-break-spaces"
+                        id="stress-log"></div>
+                </div>
+            </div>
+
+            <div class="px-4 py-2 border-t border-border text-sm text-secondary">
+                <div class="flex justify-between">
+                    <span>Completed: <span id="stress-completed">0</span>/<span id="stress-total">0</span></span>
+                    <span>Avg. Time: <span id="stress-avg-time">0</span>ms</span>
+                    <span>Peak Memory: <span id="stress-peak-mem">0</span>MB</span>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <script>
+        // Tab switching
+        document.querySelectorAll('.tab-btn').forEach(btn => {
+            btn.addEventListener('click', () => {
+                document.querySelectorAll('.tab-btn').forEach(b => b.classList.remove('active'));
+                document.querySelectorAll('.tab-content').forEach(c => c.classList.add('hidden'));
+
+                btn.classList.add('active');
+                const tabName = btn.dataset.tab;
+                document.querySelector(`#${tabName}-content`).parentElement.classList.remove('hidden');
+
+                // Re-highlight content when switching tabs
+                const activeCode = document.querySelector(`#${tabName}-content code`);
+                if (activeCode) {
+                    forceHighlightElement(activeCode);
+                }
+            });
+        });
+
+        // View switching
+        document.getElementById('play-tab').addEventListener('click', () => {
+            document.getElementById('playground').classList.remove('hidden');
+            document.getElementById('stress-modal').classList.add('hidden');
+            document.getElementById('play-tab').classList.add('bg-surface', 'border-b-0');
+            document.getElementById('stress-tab').classList.remove('bg-surface', 'border-b-0');
+        });
+
+        document.getElementById('stress-tab').addEventListener('click', () => {
+            document.getElementById('stress-modal').classList.remove('hidden');
+            document.getElementById('stress-tab').classList.add('bg-surface', 'border-b-0');
+            document.getElementById('play-tab').classList.remove('bg-surface', 'border-b-0');
+        });
+
+        document.getElementById('close-stress').addEventListener('click', () => {
+            document.getElementById('stress-modal').classList.add('hidden');
+            document.getElementById('play-tab').classList.add('bg-surface', 'border-b-0');
+            document.getElementById('stress-tab').classList.remove('bg-surface', 'border-b-0');
+        });
+
+        // Initialize clipboard and highlight.js
+        new ClipboardJS('#export-btn');
+        hljs.highlightAll();
+
+        // Keyboard shortcut
+        window.addEventListener('keydown', e => {
+            if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') {
+                document.getElementById('run-btn').click();
+            }
+        });
+
+        // ================ ADVANCED CONFIG EDITOR ================
+        const cm = CodeMirror(document.getElementById('adv-editor'), {
+            value: `CrawlerRunConfig(
+    stream=True,
+    cache_mode=CacheMode.BYPASS,
+)`,
+            mode: 'python',
+            lineNumbers: true,
+            theme: 'darcula',
+            tabSize: 4,
+            styleActiveLine: true,
+            matchBrackets: true,
+            gutters: ["CodeMirror-linenumbers"],
+            lineWrapping: true,
+        });
+
+        const TEMPLATES = {
+            CrawlerRunConfig: `CrawlerRunConfig(
+    stream=True,
+    cache_mode=CacheMode.BYPASS,
+)`,
+            BrowserConfig: `BrowserConfig(
+    headless=True,
+    extra_args=[
+        "--no-sandbox",
+        "--disable-gpu",
+    ],
+)`,
+        };
+
+        document.getElementById('cfg-type').addEventListener('change', (e) => {
+            cm.setValue(TEMPLATES[e.target.value]);
+            document.getElementById('cfg-status').textContent = '';
+        });
+        
+        // Handle endpoint selection change to show appropriate options
+        document.getElementById('endpoint').addEventListener('change', function(e) {
+            const endpoint = e.target.value;
+            const mdOptions = document.getElementById('md-options');
+            const llmOptions = document.getElementById('llm-options');
+            const advConfig = document.getElementById('adv-config');
+            
+            // Hide all option sections first
+            mdOptions.classList.add('hidden');
+            llmOptions.classList.add('hidden');
+            advConfig.classList.add('hidden');
+            
+            // Show the appropriate section based on endpoint
+            if (endpoint === 'md') {
+                mdOptions.classList.remove('hidden');
+                // Auto-open the /md options
+                mdOptions.setAttribute('open', '');
+            } else if (endpoint === 'llm') {
+                llmOptions.classList.remove('hidden');
+                // Auto-open the /llm options
+                llmOptions.setAttribute('open', '');
+            } else {
+                // For /crawl endpoints, show the advanced config
+                advConfig.classList.remove('hidden');
+            }
+        });
+
+        async function pyConfigToJson() {
+            const code = cm.getValue().trim();
+            if (!code) return {};
+
+            const res = await fetch('/config/dump', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ code }),
+            });
+
+            const statusEl = document.getElementById('cfg-status');
+            if (!res.ok) {
+                const msg = await res.text();
+                statusEl.textContent = '✖ config error';
+                statusEl.className = 'text-xs text-red-400';
+                throw new Error(msg || 'Invalid config');
+            }
+
+            statusEl.textContent = '✓ parsed';
+            statusEl.className = 'text-xs text-green-400';
+
+            return await res.json();
+        }
+
+        // ================ SERVER COMMUNICATION ================
+
+        // Update status UI
+        function updateStatus(status, time, memory, peakMemory) {
+            const statusEl = document.getElementById('execution-status');
+            const badgeEl = document.querySelector('#status-badge span:first-child');
+            const textEl = document.querySelector('#status-badge span:last-child');
+
+            statusEl.classList.remove('hidden');
+            badgeEl.className = 'w-3 h-3 rounded-full mr-2';
+
+            if (status === 'success') {
+                badgeEl.classList.add('bg-green-500');
+                textEl.textContent = 'Success';
+            } else if (status === 'error') {
+                badgeEl.classList.add('bg-red-500');
+                textEl.textContent = 'Error';
+            } else {
+                badgeEl.classList.add('bg-yellow-500');
+                textEl.textContent = 'Processing...';
+            }
+
+            if (time) {
+                document.getElementById('exec-time').textContent = `${time}ms`;
+            }
+
+            if (memory !== undefined && peakMemory !== undefined) {
+                document.getElementById('exec-mem').textContent = `Δ${memory >= 0 ? '+' : ''}${memory}MB (Peak: ${peakMemory}MB)`;
+            }
+        }
+
+        // Generate code snippets
+        function generateSnippets(api, payload, method = 'POST') {
+            // Python snippet
+            const pyCodeEl = document.querySelector('#python-content code');
+            let pySnippet;
+            
+            if (method === 'GET') {
+                // GET request (for /llm endpoint)
+                pySnippet = `import httpx\n\nasync def crawl():\n    async with httpx.AsyncClient() as client:\n        response = await client.get(\n            "${window.location.origin}${api}"\n        )\n        return response.json()`;
+            } else {
+                // POST request (for /crawl and /md endpoints)
+                pySnippet = `import httpx\n\nasync def crawl():\n    async with httpx.AsyncClient() as client:\n        response = await client.post(\n            "${window.location.origin}${api}",\n            json=${JSON.stringify(payload, null, 4).replace(/\n/g, '\n            ')}\n        )\n        return response.json()`;
+            }
+
+            pyCodeEl.textContent = pySnippet;
+            pyCodeEl.className = 'python hljs'; // Reset classes
+            forceHighlightElement(pyCodeEl);
+
+            // cURL snippet
+            const curlCodeEl = document.querySelector('#curl-content code');
+            let curlSnippet;
+            
+            if (method === 'GET') {
+                // GET request (for /llm endpoint)
+                curlSnippet = `curl -X GET "${window.location.origin}${api}"`;
+            } else {
+                // POST request (for /crawl and /md endpoints)
+                curlSnippet = `curl -X POST ${window.location.origin}${api} \\\n  -H "Content-Type: application/json" \\\n  -d '${JSON.stringify(payload)}'`;
+            }
+
+            curlCodeEl.textContent = curlSnippet;
+            curlCodeEl.className = 'bash hljs'; // Reset classes
+            forceHighlightElement(curlCodeEl);
+        }
+
+        // Main run function
+        async function runCrawl() {
+            const endpoint = document.getElementById('endpoint').value;
+            const urls = document.getElementById('urls').value.trim().split(/\n/).filter(u => u);
+            // 1) grab python from CodeMirror, validate via /config/dump
+            let advConfig = {};
+            try {
+                const cfgJson = await pyConfigToJson(); // may throw
+                if (Object.keys(cfgJson).length) {
+                    const cfgType = document.getElementById('cfg-type').value;
+                    advConfig = cfgType === 'CrawlerRunConfig'
+                        ? { crawler_config: cfgJson }
+                        : { browser_config: cfgJson };
+                }
+            } catch (err) {
+                updateStatus('error');
+                document.querySelector('#response-content code').textContent =
+                    JSON.stringify({ error: err.message }, null, 2);
+                forceHighlightElement(document.querySelector('#response-content code'));
+                return; // stop run
+            }
+
+            const endpointMap = {
+                crawl: '/crawl',
+                // crawl_stream: '/crawl/stream',
+                md: '/md',
+                llm: '/llm'
+            };
+
+            const api = endpointMap[endpoint];
+            let payload;
+            
+            // Create appropriate payload based on endpoint type
+            if (endpoint === 'md') {
+                // Get values from the /md specific inputs
+                const filterType = document.getElementById('md-filter').value;
+                const query = document.getElementById('md-query').value.trim();
+                const cache = document.getElementById('md-cache').value;
+                
+                // MD endpoint expects: { url, f, q, c }
+                payload = {
+                    url: urls[0], // Take first URL
+                    f: filterType, // Lowercase filter type as required by server
+                    q: query || null, // Use the query if provided, otherwise null
+                    c: cache
+                };
+            } else if (endpoint === 'llm') {
+                // LLM endpoint has a different URL pattern and uses query params
+                // This will be handled directly in the fetch below
+                payload = null;
+            } else {
+                // Default payload for /crawl and /crawl/stream
+                payload = {
+                    urls,
+                    ...advConfig
+                };
+            }
+
+            updateStatus('processing');
+
+            try {
+                const startTime = performance.now();
+                let response, responseData;
+
+                if (endpoint === 'llm') {
+                    // Special handling for LLM endpoint which uses URL pattern: /llm/{encoded_url}?q={query}
+                    const url = urls[0];
+                    const encodedUrl = encodeURIComponent(url);
+                    // Get the question from the LLM-specific input
+                    const question = document.getElementById('llm-question').value.trim() || "What is this page about?";
+                    
+                    response = await fetch(`${api}/${encodedUrl}?q=${encodeURIComponent(question)}`, {
+                        method: 'GET',
+                        headers: { 'Accept': 'application/json' }
+                    });
+                    responseData = await response.json();
+                    const time = Math.round(performance.now() - startTime);
+                    if (!response.ok) {
+                        updateStatus('error', time);
+                        throw new Error(responseData.error || 'Request failed');
+                    }
+                    updateStatus('success', time);
+                    document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2);
+                    document.querySelector('#response-content code').className = 'json hljs';
+                    forceHighlightElement(document.querySelector('#response-content code'));
+                } else if (endpoint === 'crawl_stream') {
+                    // Stream processing
+                    response = await fetch(api, {
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify(payload)
+                    });
+
+                    const reader = response.body.getReader();
+                    let text = '';
+                    let maxMemory = 0;
+
+                    while (true) {
+                        const { value, done } = await reader.read();
+                        if (done) break;
+
+                        const chunk = new TextDecoder().decode(value);
+                        text += chunk;
+
+                        // Process each line for memory updates
+                        chunk.trim().split('\n').forEach(line => {
+                            if (!line) return;
+                            try {
+                                const obj = JSON.parse(line);
+                                if (obj.server_memory_mb) {
+                                    maxMemory = Math.max(maxMemory, obj.server_memory_mb);
+                                }
+                            } catch (e) {
+                                console.error('Error parsing stream line:', e);
+                            }
+                        });
+                    }
+
+                    responseData = { stream: text };
+                    const time = Math.round(performance.now() - startTime);
+                    updateStatus('success', time, null, maxMemory);
+                    document.querySelector('#response-content code').textContent = text;
+                    document.querySelector('#response-content code').className = 'json hljs'; // Reset classes
+                    forceHighlightElement(document.querySelector('#response-content code'));
+                } else {
+                    // Regular request (handles /crawl and /md)
+                    response = await fetch(api, {
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify(payload)
+                    });
+
+                    responseData = await response.json();
+                    const time = Math.round(performance.now() - startTime);
+
+                    if (!response.ok) {
+                        updateStatus('error', time);
+                        throw new Error(responseData.error || 'Request failed');
+                    }
+
+                    updateStatus(
+                        'success',
+                        time,
+                        responseData.server_memory_delta_mb,
+                        responseData.server_peak_memory_mb
+                    );
+
+                    document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2);
+                    document.querySelector('#response-content code').className = 'json hljs'; // Ensure class is set
+                    forceHighlightElement(document.querySelector('#response-content code'));
+                }
+
+                forceHighlightElement(document.querySelector('#response-content code'));
+                
+                // For generateSnippets, handle the LLM case specially
+                if (endpoint === 'llm') {
+                    const url = urls[0];
+                    const encodedUrl = encodeURIComponent(url);
+                    const question = document.getElementById('llm-question').value.trim() || "What is this page about?";
+                    generateSnippets(`${api}/${encodedUrl}?q=${encodeURIComponent(question)}`, null, 'GET');
+                } else {
+                    generateSnippets(api, payload);
+                }
+            } catch (error) {
+                console.error('Error:', error);
+                updateStatus('error');
+                document.querySelector('#response-content code').textContent = JSON.stringify(
+                    { error: error.message },
+                    null,
+                    2
+                );
+                forceHighlightElement(document.querySelector('#response-content code'));
+            }
+        }
+
+        // Stress test function
+        async function runStressTest() {
+            const total = parseInt(document.getElementById('st-total').value);
+            const chunkSize = parseInt(document.getElementById('st-chunk').value);
+            const concurrency = parseInt(document.getElementById('st-conc').value);
+            const useStream = document.getElementById('st-stream').checked;
+
+            const logEl = document.getElementById('stress-log');
+            logEl.textContent = '';
+
+            document.getElementById('stress-completed').textContent = '0';
+            document.getElementById('stress-total').textContent = total;
+            document.getElementById('stress-avg-time').textContent = '0';
+            document.getElementById('stress-peak-mem').textContent = '0';
+
+            const api = useStream ? '/crawl/stream' : '/crawl';
+            const urls = Array.from({ length: total }, (_, i) => `https://httpbin.org/anything/stress-${i}-${Date.now()}`);
+            const chunks = [];
+
+            for (let i = 0; i < urls.length; i += chunkSize) {
+                chunks.push(urls.slice(i, i + chunkSize));
+            }
+
+            let completed = 0;
+            let totalTime = 0;
+            let peakMemory = 0;
+
+            const processBatch = async (batch, index) => {
+                const payload = {
+                    urls: batch,
+                    browser_config: {},
+                    crawler_config: { cache_mode: 'BYPASS', stream: useStream }
+                };
+
+                const start = performance.now();
+                let time, memory;
+
+                try {
+                    if (useStream) {
+                        const response = await fetch(api, {
+                            method: 'POST',
+                            headers: { 'Content-Type': 'application/json' },
+                            body: JSON.stringify(payload)
+                        });
+
+                        const reader = response.body.getReader();
+                        let maxMem = 0;
+                        while (true) {
+                            const { value, done } = await reader.read();
+                            if (done) break;
+                            const text = new TextDecoder().decode(value);
+                            text.split('\n').forEach(line => {
+                                try {
+                                    const obj = JSON.parse(line);
+                                    if (obj.server_memory_mb) {
+                                        maxMem = Math.max(maxMem, obj.server_memory_mb);
+                                    }
+                                } catch { }
+                            });
+                        }
+
+                        memory = maxMem;
+                    } else {
+                        const response = await fetch(api, {
+                            method: 'POST',
+                            headers: { 'Content-Type': 'application/json' },
+                            body: JSON.stringify(payload)
+                        });
+
+                        const data = await response.json();
+                        memory = data.server_peak_memory_mb;
+                    }
+
+                    time = Math.round(performance.now() - start);
+                    peakMemory = Math.max(peakMemory, memory || 0);
+                    totalTime += time;
+
+                    logEl.textContent += `[${index + 1}/${chunks.length}] ✔ ${time}ms | Peak ${memory}MB\n`;
+                } catch (error) {
+                    time = Math.round(performance.now() - start);
+                    logEl.textContent += `[${index + 1}/${chunks.length}] ✖ ${time}ms | ${error.message}\n`;
+                }
+
+                completed += batch.length;
+                document.getElementById('stress-completed').textContent = completed;
+                document.getElementById('stress-peak-mem').textContent = peakMemory;
+                document.getElementById('stress-avg-time').textContent = Math.round(totalTime / (index + 1));
+
+                logEl.scrollTop = logEl.scrollHeight;
+            };
+
+            // Run with concurrency control
+            let active = 0;
+            let index = 0;
+
+            return new Promise(resolve => {
+                const runNext = () => {
+                    while (active < concurrency && index < chunks.length) {
+                        processBatch(chunks[index], index)
+                            .finally(() => {
+                                active--;
+                                runNext();
+                            });
+                        active++;
+                        index++;
+                    }
+
+                    if (active === 0 && index >= chunks.length) {
+                        logEl.textContent += '\n✅ Stress test completed\n';
+                        resolve();
+                    }
+                };
+
+                runNext();
+            });
+        }
+
+        // Event listeners
+        document.getElementById('run-btn').addEventListener('click', runCrawl);
+        document.getElementById('st-run').addEventListener('click', runStressTest);
+
+        function forceHighlightElement(element) {
+            if (!element) return;
+
+            // Save current scroll position (important for large code blocks)
+            const scrollTop = element.parentElement.scrollTop;
+
+            // Reset the element
+            const text = element.textContent;
+            element.innerHTML = text;
+            element.removeAttribute('data-highlighted');
+
+            // Reapply highlighting
+            hljs.highlightElement(element);
+
+            // Restore scroll position
+            element.parentElement.scrollTop = scrollTop;
+        }
+
+        // Initialize clipboard for all copy buttons
+        function initCopyButtons() {
+            document.querySelectorAll('.copy-btn').forEach(btn => {
+                new  ClipboardJS(btn, {
+                    text: () => {
+                        const target = document.querySelector(btn.dataset.target);
+                        return target ? target.textContent : '';
+                    }
+                }).on('success', e => {
+                    e.clearSelection();
+                    // make button text "copied" for 1 second
+                    const originalText = e.trigger.textContent;
+                    e.trigger.textContent = 'Copied!';
+                    setTimeout(() => {
+                        e.trigger.textContent = originalText;
+                    }, 1000);
+                    // Highlight the copied code
+                    const target = document.querySelector(btn.dataset.target);
+                    if (target) {
+                        target.classList.add('highlighted');
+                        setTimeout(() => {
+                            target.classList.remove('highlighted');
+                        }, 1000);
+                    }
+
+                }).on('error', e => {
+                    console.error('Error copying:', e);
+                });
+            });
+        }
+        
+        // Function to initialize UI based on selected endpoint
+        function initUI() {
+            // Trigger the endpoint change handler to set initial UI state
+            const endpointSelect = document.getElementById('endpoint');
+            const event = new Event('change');
+            endpointSelect.dispatchEvent(event);
+            
+            // Initialize copy buttons
+            initCopyButtons();
+        }
+
+        // Initialize on page load
+        document.addEventListener('DOMContentLoaded', initUI);
+        // Also call it immediately in case the script runs after DOM is already loaded
+        if (document.readyState !== 'loading') {
+            initUI();
+        }
+
+    </script>
+</body>
+
+</html>

deploy/docker/supervisord.conf

@@ -1,12 +1,28 @@
 [supervisord]
-nodaemon=true
+nodaemon=true                   ; Run supervisord in the foreground
+logfile=/dev/null               ; Log supervisord output to stdout/stderr
+logfile_maxbytes=0
 
 [program:redis]
-command=redis-server
+command=/usr/bin/redis-server --loglevel notice ; Path to redis-server on Alpine
+user=appuser                    ; Run redis as our non-root user
 autorestart=true
 priority=10
+stdout_logfile=/dev/stdout      ; Redirect redis stdout to container stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr      ; Redirect redis stderr to container stderr
+stderr_logfile_maxbytes=0
 
 [program:gunicorn]
-command=gunicorn --bind 0.0.0.0:8000 --workers 4 --threads 2 --timeout 300 --graceful-timeout 60 --keep-alive 65 --log-level debug --worker-class uvicorn.workers.UvicornWorker --max-requests 1000 --max-requests-jitter 50 server:app
+command=/usr/local/bin/gunicorn --bind 0.0.0.0:11235 --workers 1 --threads 4 --timeout 1800 --graceful-timeout 30 --keep-alive 300 --log-level info --worker-class uvicorn.workers.UvicornWorker server:app
+directory=/app                  ; Working directory for the app
+user=appuser                    ; Run gunicorn as our non-root user
 autorestart=true
-priority=20
+priority=20
+environment=PYTHONUNBUFFERED=1  ; Ensure Python output is sent straight to logs
+stdout_logfile=/dev/stdout      ; Redirect gunicorn stdout to container stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr      ; Redirect gunicorn stderr to container stderr
+stderr_logfile_maxbytes=0
+
+# Optional: Add filebeat or other logging agents here if needed

deploy/docker/utils.py

@@ -1,6 +1,7 @@
 import dns.resolver
 import logging
 import yaml
+import os
 from datetime import datetime
 from enum import Enum
 from pathlib import Path
@@ -19,10 +20,24 @@ class FilterType(str, Enum):
     LLM = "llm"
 
 def load_config() -> Dict:
-    """Load and return application configuration."""
+    """Load and return application configuration with environment variable overrides."""
     config_path = Path(__file__).parent / "config.yml"
     with open(config_path, "r") as config_file:
-        return yaml.safe_load(config_file)
+        config = yaml.safe_load(config_file)
+    
+    # Override LLM provider from environment if set
+    llm_provider = os.environ.get("LLM_PROVIDER")
+    if llm_provider:
+        config["llm"]["provider"] = llm_provider
+        logging.info(f"LLM provider overridden from environment: {llm_provider}")
+    
+    # Also support direct API key from environment if the provider-specific key isn't set
+    llm_api_key = os.environ.get("LLM_API_KEY")
+    if llm_api_key and "api_key" not in config["llm"]:
+        config["llm"]["api_key"] = llm_api_key
+        logging.info("LLM API key loaded from LLM_API_KEY environment variable")
+    
+    return config
 
 def setup_logging(config: Dict) -> None:
     """Configure application logging."""
@@ -45,10 +60,10 @@ def datetime_handler(obj: any) -> Optional[str]:
         return obj.isoformat()
     raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
 
-def should_cleanup_task(created_at: str) -> bool:
+def should_cleanup_task(created_at: str, ttl_seconds: int = 3600) -> bool:
     """Check if task should be cleaned up based on creation time."""
     created = datetime.fromisoformat(created_at)
-    return (datetime.now() - created).total_seconds() > 3600
+    return (datetime.now() - created).total_seconds() > ttl_seconds
 
 def decode_redis_hash(hash_data: Dict[bytes, bytes]) -> Dict[str, str]:
     """Decode Redis hash data from bytes to strings."""
@@ -56,6 +71,52 @@ def decode_redis_hash(hash_data: Dict[bytes, bytes]) -> Dict[str, str]:
 
 
 
+def get_llm_api_key(config: Dict, provider: Optional[str] = None) -> str:
+    """Get the appropriate API key based on the LLM provider.
+    
+    Args:
+        config: The application configuration dictionary
+        provider: Optional provider override (e.g., "openai/gpt-4")
+    
+    Returns:
+        The API key for the provider, or empty string if not found
+    """
+        
+    # Use provided provider or fall back to config
+    if not provider:
+        provider = config["llm"]["provider"]
+    
+    # Check if direct API key is configured
+    if "api_key" in config["llm"]:
+        return config["llm"]["api_key"]
+    
+    # Fall back to the configured api_key_env if no match
+    return os.environ.get(config["llm"].get("api_key_env", ""), "")
+
+
+def validate_llm_provider(config: Dict, provider: Optional[str] = None) -> tuple[bool, str]:
+    """Validate that the LLM provider has an associated API key.
+    
+    Args:
+        config: The application configuration dictionary
+        provider: Optional provider override (e.g., "openai/gpt-4")
+    
+    Returns:
+        Tuple of (is_valid, error_message)
+    """
+    # Use provided provider or fall back to config
+    if not provider:
+        provider = config["llm"]["provider"]
+    
+    # Get the API key for this provider
+    api_key = get_llm_api_key(config, provider)
+    
+    if not api_key:
+        return False, f"No API key found for provider '{provider}'. Please set the appropriate environment variable."
+    
+    return True, ""
+
+
 def verify_email_domain(email: str) -> bool:
     try:
         domain = email.split('@')[1]

docker-compose.yml

@@ -1,16 +1,22 @@
-# Base configuration (not a service, just a reusable config block)
+version: '3.8'
+
+# Shared configuration for all environments
 x-base-config: &base-config
   ports:
-    - "11235:11235"
-    - "8000:8000"
-    - "9222:9222"
-    - "8080:8080"
+    - "11235:11235"  # Gunicorn port
+  env_file:
+    - .llm.env       # API keys (create from .llm.env.example)
   environment:
-    - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
     - OPENAI_API_KEY=${OPENAI_API_KEY:-}
-    - CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
+    - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
+    - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
+    - GROQ_API_KEY=${GROQ_API_KEY:-}
+    - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
+    - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
+    - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-}
+    - LLM_PROVIDER=${LLM_PROVIDER:-}  # Optional: Override default provider (e.g., "anthropic/claude-3-opus")
   volumes:
-    - /dev/shm:/dev/shm
+    - /dev/shm:/dev/shm  # Chromium performance
   deploy:
     resources:
       limits:
@@ -24,42 +30,21 @@ x-base-config: &base-config
     timeout: 10s
     retries: 3
     start_period: 40s
+  user: "appuser"
 
 services:
-  # Local build services for different platforms
-  crawl4ai-amd64:
+  crawl4ai:
+    # 1. Default: Pull multi-platform test image from Docker Hub
+    # 2. Override with local image via: IMAGE=local-test docker compose up
+    image: ${IMAGE:-unclecode/crawl4ai:${TAG:-latest}}
+    
+    # Local build config (used with --build)
     build:
       context: .
       dockerfile: Dockerfile
       args:
-        PYTHON_VERSION: "3.10"
-        INSTALL_TYPE: ${INSTALL_TYPE:-basic}
-        ENABLE_GPU: false
-      platforms:
-        - linux/amd64
-    profiles: ["local-amd64"]
-    <<: *base-config  # extends yerine doğrudan yapılandırmayı dahil ettik
-
-  crawl4ai-arm64:
-    build:
-      context: .
-      dockerfile: Dockerfile
-      args:
-        PYTHON_VERSION: "3.10"
-        INSTALL_TYPE: ${INSTALL_TYPE:-basic}
-        ENABLE_GPU: false
-      platforms:
-        - linux/arm64
-    profiles: ["local-arm64"]
-    <<: *base-config
-
-  # Hub services for different platforms and versions
-  crawl4ai-hub-amd64:
-    image: unclecode/crawl4ai:${VERSION:-basic}-amd64
-    profiles: ["hub-amd64"]
-    <<: *base-config
-
-  crawl4ai-hub-arm64:
-    image: unclecode/crawl4ai:${VERSION:-basic}-arm64
-    profiles: ["hub-arm64"]
+        INSTALL_TYPE: ${INSTALL_TYPE:-default}
+        ENABLE_GPU: ${ENABLE_GPU:-false}
+    
+    # Inherit shared config
     <<: *base-config

docs/apps/linkdin/README.md

@@ -0,0 +1,131 @@
+# Crawl4AI Prospect‑Wizard – step‑by‑step guide
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/10nRCwmfxPjVrRUHyJsYlX7BH5bvPoGpx?usp=sharing)
+
+A three‑stage demo that goes from **LinkedIn scraping** ➜ **LLM reasoning** ➜ **graph visualisation**.
+
+**Try it in Google Colab!** Click the badge above to run this demo in a cloud environment with zero setup required.
+
+```
+prospect‑wizard/
+├─ c4ai_discover.py         # Stage 1 – scrape companies + people
+├─ c4ai_insights.py         # Stage 2 – embeddings, org‑charts, scores
+├─ graph_view_template.html # Stage 3 – graph viewer (static HTML)
+└─ data/                    # output lands here (*.jsonl / *.json)
+```
+
+---
+
+## 1  Install & boot a LinkedIn profile (one‑time)
+
+### 1.1  Install dependencies
+```bash
+pip install crawl4ai litellm sentence-transformers pandas rich
+```
+
+### 1.2  Create / warm a LinkedIn browser profile
+```bash
+crwl profiles
+```
+1. The interactive shell shows **New profile** – hit **enter**.
+2. Choose a name, e.g. `profile_linkedin_uc`.
+3. A Chromium window opens – log in to LinkedIn, solve whatever CAPTCHA, then close.
+
+> Remember the **profile name**. All future runs take `--profile-name <your_name>`.
+
+---
+
+## 2  Discovery – scrape companies & people
+
+```bash
+python c4ai_discover.py full \
+  --query "health insurance management" \
+  --geo 102713980 \               # Malaysia geoUrn
+  --title-filters "" \            # or "Product,Engineering"
+  --max-companies 10 \            # default set small for workshops
+  --max-people 20 \               # \^ same
+  --profile-name profile_linkedin_uc \
+  --outdir ./data \
+  --concurrency 2 \
+  --log-level debug
+```
+**Outputs** in `./data/`:
+* `companies.jsonl` – one JSON per company
+* `people.jsonl` – one JSON per employee
+
+🛠️  **Dry‑run:** `C4AI_DEMO_DEBUG=1 python c4ai_discover.py full --query coffee` uses bundled HTML snippets, no network.
+
+### Handy geoUrn cheatsheet
+| Location | geoUrn |
+|----------|--------|
+| Singapore | **103644278** |
+| Malaysia | **102713980** |
+| United States | **103644922** |
+| United Kingdom | **102221843** |
+| Australia | **101452733** |
+_See more: <https://www.linkedin.com/search/results/companies/?geoUrn=XXX> – the number after `geoUrn=` is what you need._
+
+---
+
+## 3  Insights – embeddings, org‑charts, decision makers
+
+```bash
+python c4ai_insights.py \
+  --in ./data \
+  --out ./data \
+  --embed-model all-MiniLM-L6-v2 \
+  --llm-provider gemini/gemini-2.0-flash \
+  --llm-api-key "" \
+  --top-k 10 \
+  --max-llm-tokens 8024 \
+  --llm-temperature 1.0 \
+  --workers 4
+```
+Emits next to the Stage‑1 files:
+* `company_graph.json` – inter‑company similarity graph
+* `org_chart_<handle>.json` – one per company
+* `decision_makers.csv` – hand‑picked ‘who to pitch’ list
+
+Flags reference (straight from `build_arg_parser()`):
+| Flag | Default | Purpose |
+|------|---------|---------|
+| `--in` | `.` | Stage‑1 output dir |
+| `--out` | `.` | Destination dir |
+| `--embed_model` | `all-MiniLM-L6-v2` | Sentence‑Transformer model |
+| `--top_k` | `10` | Neighbours per company in graph |
+| `--openai_model` | `gpt-4.1` | LLM for scoring decision makers |
+| `--max_llm_tokens` | `8024` | Token budget per LLM call |
+| `--llm_temperature` | `1.0` | Creativity knob |
+| `--stub` | off | Skip OpenAI and fabricate tiny charts |
+| `--workers` | `4` | Parallel LLM workers |
+
+---
+
+## 4  Visualise – interactive graph
+
+After Stage 2 completes, simply open the HTML viewer from the project root:
+```bash
+open graph_view_template.html   # or Live Server / Python -http
+```
+The page fetches `data/company_graph.json` and the `org_chart_*.json` files automatically; keep the `data/` folder beside the HTML file.
+
+* Left pane → list of companies (clans).
+* Click a node to load its org‑chart on the right.
+* Chat drawer lets you ask follow‑up questions; context is pulled from `people.jsonl`.
+
+---
+
+## 5  Common snags
+
+| Symptom | Fix |
+|---------|-----|
+| Infinite CAPTCHA | Use a residential proxy: `--proxy http://user:pass@ip:port` |
+| 429 Too Many Requests | Lower `--concurrency`, rotate profile, add delay |
+| Blank graph | Check JSON paths, clear `localStorage` in browser |
+
+---
+
+### TL;DR
+`crwl profiles` → `c4ai_discover.py` → `c4ai_insights.py` → open `graph_view_template.html`.  
+Live long and `import crawl4ai`.
+

docs/apps/linkdin/c4ai_discover.py

@@ -0,0 +1,446 @@
+#!/usr/bin/env python3
+"""
+c4ai-discover — Stage‑1 Discovery CLI
+
+Scrapes LinkedIn company search + their people pages and dumps two newline‑delimited
+JSON files: companies.jsonl and people.jsonl.
+
+Key design rules
+----------------
+* No BeautifulSoup — Crawl4AI only for network + HTML fetch.
+* JsonCssExtractionStrategy for structured scraping; schema auto‑generated once
+  from sample HTML provided by user and then cached under ./schemas/.
+* Defaults are embedded so the file runs inside VS Code debugger without CLI args.
+* If executed as a console script (argv > 1), CLI flags win.
+* Lightweight deps: argparse + Crawl4AI stack.
+
+Author: Tom @ Kidocode 2025‑04‑26
+"""
+from __future__ import annotations
+
+import warnings, re
+warnings.filterwarnings(
+    "ignore",
+    message=r"The pseudo class ':contains' is deprecated, ':-soup-contains' should be used.*",
+    category=FutureWarning,
+    module=r"soupsieve"
+)
+
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Imports
+# ───────────────────────────────────────────────────────────────────────────────
+import argparse
+import random
+import asyncio
+import json
+import logging
+import os
+import pathlib
+import sys
+# 3rd-party rich for pretty logging
+from rich.console import Console
+from rich.logging import RichHandler
+
+from datetime import datetime, UTC
+from textwrap import dedent
+from types import SimpleNamespace
+from typing import Dict, List, Optional
+from urllib.parse import quote
+from pathlib import Path
+from glob import glob
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    BrowserConfig,
+    CacheMode,
+    CrawlerRunConfig,
+    JsonCssExtractionStrategy,
+    BrowserProfiler,
+    LLMConfig,
+)
+
+# ───────────────────────────────────────────────────────────────────────────────
+# Constants / paths
+# ───────────────────────────────────────────────────────────────────────────────
+BASE_DIR = pathlib.Path(__file__).resolve().parent
+SCHEMA_DIR = BASE_DIR / "schemas"
+SCHEMA_DIR.mkdir(parents=True, exist_ok=True)
+COMPANY_SCHEMA_PATH = SCHEMA_DIR / "company_card.json"
+PEOPLE_SCHEMA_PATH = SCHEMA_DIR / "people_card.json"
+
+# ---------- deterministic target JSON examples ----------
+_COMPANY_SCHEMA_EXAMPLE = {
+    "handle": "/company/posify/",
+    "profile_image": "https://media.licdn.com/dms/image/v2/.../logo.jpg",
+    "name": "Management Research Services, Inc. (MRS, Inc)",
+    "descriptor": "Insurance • Milwaukee, Wisconsin",
+    "about": "Insurance • Milwaukee, Wisconsin",
+    "followers": 1000
+}
+
+_PEOPLE_SCHEMA_EXAMPLE = {
+    "profile_url": "https://www.linkedin.com/in/lily-ng/",
+    "name": "Lily Ng",
+    "headline": "VP Product @ Posify",
+    "followers": 890,
+    "connection_degree": "2nd",
+    "avatar_url": "https://media.licdn.com/dms/image/v2/.../lily.jpg"
+}
+
+# Provided sample HTML snippets (trimmed) — used exactly once to cold‑generate schema.
+_SAMPLE_COMPANY_HTML = (Path(__file__).resolve().parent / "snippets/company.html").read_text()
+_SAMPLE_PEOPLE_HTML = (Path(__file__).resolve().parent / "snippets/people.html").read_text()
+
+# --------- tighter schema prompts ----------
+_COMPANY_SCHEMA_QUERY = dedent(
+    """
+    Using the supplied <li> company-card HTML, build a JsonCssExtractionStrategy schema that,
+    for every card, outputs *exactly* the keys shown in the example JSON below.
+    JSON spec:
+      • handle        – href of the outermost <a> that wraps the logo/title, e.g. "/company/posify/"
+      • profile_image – absolute URL of the <img> inside that link
+      • name          – text of the <a> inside the <span class*='t-16'>
+      • descriptor    – text line with industry • location
+      • about         – text of the <div class*='t-normal'> below the name (industry + geo)
+      • followers     – integer parsed from the <div> containing 'followers'
+      
+    IMPORTANT: Do not use the base64 kind of classes to target element. It's not reliable.
+    The main div parent contains these li element is "div.search-results-container" you can use this.
+    The <ul> parent has "role" equal to "list". Using these two should be enough to target the <li> elements.    
+
+    IMPORTANT: Remember there might be multiple <a> tags that start with https://www.linkedin.com/company/[NAME], 
+    so in case you refer to them for different fields, make sure to be more specific. One has the image, and one 
+    has the person's name.
+    
+    IMPORTANT: Be very smart in selecting the correct and unique way to address the element. You should ensure 
+    your selector points to a single element and is unique to the place that contains the information.
+    """
+)
+
+_PEOPLE_SCHEMA_QUERY = dedent(
+    """
+    Using the supplied <li> people-card HTML, build a JsonCssExtractionStrategy schema that
+    outputs exactly the keys in the example JSON below.
+    Fields:
+      • profile_url        – href of the outermost profile link
+      • name               – text inside artdeco-entity-lockup__title
+      • headline           – inner text of artdeco-entity-lockup__subtitle
+      • followers          – integer parsed from the span inside lt-line-clamp--multi-line
+      • connection_degree  – '1st', '2nd', etc. from artdeco-entity-lockup__badge
+      • avatar_url         – src of the <img> within artdeco-entity-lockup__image
+      
+    IMPORTANT: Do not use the base64 kind of classes to target element. It's not reliable.
+    The main div parent contains these li element is a "div" has these classes "artdeco-card org-people-profile-card__card-spacing org-people__card-margin-bottom".
+    """
+)
+
+# ---------------------------------------------------------------------------
+# Utility helpers
+# ---------------------------------------------------------------------------
+
+def _load_or_build_schema(
+    path: pathlib.Path, 
+    sample_html: str, 
+    query: str, 
+    example_json: Dict,
+    force = False
+) -> Dict:
+    """Load schema from path, else call generate_schema once and persist."""
+    if path.exists() and not force:
+        return json.loads(path.read_text())
+
+    logging.info("[SCHEMA] Generating schema %s", path.name)
+    schema = JsonCssExtractionStrategy.generate_schema(
+        html=sample_html,
+        llm_config=LLMConfig(
+            provider=os.getenv("C4AI_SCHEMA_PROVIDER", "openai/gpt-4o"),
+            api_token=os.getenv("OPENAI_API_KEY", "env:OPENAI_API_KEY"),
+        ),
+        query=query,
+        target_json_example=json.dumps(example_json, indent=2),
+    )
+    path.write_text(json.dumps(schema, indent=2))
+    return schema
+
+
+def _openai_friendly_number(text: str) -> Optional[int]:
+    """Extract first int from text like '1K followers' (returns 1000)."""
+    import re
+
+    m = re.search(r"(\d[\d,]*)", text.replace(",", ""))
+    if not m:
+        return None
+    val = int(m.group(1))
+    if "k" in text.lower():
+        val *= 1000
+    if "m" in text.lower():
+        val *= 1_000_000
+    return val
+
+# ---------------------------------------------------------------------------
+# Core async workers
+# ---------------------------------------------------------------------------
+async def crawl_company_search(crawler: AsyncWebCrawler, url: str, schema: Dict, limit: int) -> List[Dict]:
+    """Paginate 10-item company search pages until `limit` reached."""
+    extraction = JsonCssExtractionStrategy(schema)
+    cfg = CrawlerRunConfig(
+        extraction_strategy=extraction,
+        cache_mode=CacheMode.BYPASS,
+        wait_for = ".search-marvel-srp",
+        session_id="company_search",
+        delay_before_return_html=1,
+        magic = True,
+        verbose= False,
+    )
+    companies, page = [], 1
+    while len(companies) < max(limit, 10):
+        paged_url = f"{url}&page={page}"
+        res = await crawler.arun(paged_url, config=cfg)
+        batch = json.loads(res[0].extracted_content)
+        if not batch:
+            break
+        for item in batch:
+            name = item.get("name", "").strip()
+            handle = item.get("handle", "").strip()
+            if not handle or not name:
+                continue
+            descriptor = item.get("descriptor")
+            about = item.get("about")
+            followers = _openai_friendly_number(str(item.get("followers", "")))
+            companies.append(
+                {
+                    "handle": handle,
+                    "name": name,
+                    "descriptor": descriptor,
+                    "about": about,
+                    "followers": followers,
+                    "people_url": f"{handle}people/",
+                    "captured_at": datetime.now(UTC).isoformat(timespec="seconds") + "Z",
+                }
+            )
+        page += 1
+        logging.info(
+            f"[dim]Page {page}[/] — running total: {len(companies)}/{limit} companies"
+        )
+
+    return companies[:max(limit, 10)]
+
+
+async def crawl_people_page(
+    crawler: AsyncWebCrawler,
+    people_url: str,
+    schema: Dict,
+    limit: int,
+    title_kw: str,
+) -> List[Dict]:
+    people_u = f"{people_url}?keywords={quote(title_kw)}"
+    extraction = JsonCssExtractionStrategy(schema)
+    cfg = CrawlerRunConfig(
+        extraction_strategy=extraction,
+        # scan_full_page=True,
+        cache_mode=CacheMode.BYPASS,
+        magic=True,
+        wait_for=".org-people-profile-card__card-spacing",
+        wait_for_images=5000,
+        delay_before_return_html=1,
+        session_id="people_search",
+    )
+    res = await crawler.arun(people_u, config=cfg)
+    if not res[0].success:
+        return []
+    raw = json.loads(res[0].extracted_content)
+    people = []
+    for p in raw[:limit]:
+        followers = _openai_friendly_number(str(p.get("followers", "")))
+        people.append(
+            {
+                "profile_url": p.get("profile_url"),
+                "name": p.get("name"),
+                "headline": p.get("headline"),
+                "followers": followers,
+                "connection_degree": p.get("connection_degree"),
+                "avatar_url": p.get("avatar_url"),
+            }
+        )
+    return people
+
+# ---------------------------------------------------------------------------
+# CLI + main
+# ---------------------------------------------------------------------------
+
+def build_arg_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser("c4ai-discover — Crawl4AI LinkedIn discovery")
+    sub = ap.add_subparsers(dest="cmd", required=False, help="run scope")
+
+    def add_flags(parser: argparse.ArgumentParser):
+        parser.add_argument("--query", required=False, help="query keyword(s)")
+        parser.add_argument("--geo", required=False, type=int, help="LinkedIn geoUrn")
+        parser.add_argument("--title-filters", default="Product,Engineering", help="comma list of job keywords")
+        parser.add_argument("--max-companies", type=int, default=1000)
+        parser.add_argument("--max-people", type=int, default=500)
+        parser.add_argument("--profile-name", default=str(pathlib.Path.home() / ".crawl4ai/profiles/profile_linkedin_uc"))
+        parser.add_argument("--outdir", default="./output")
+        parser.add_argument("--concurrency", type=int, default=4)
+        parser.add_argument("--log-level", default="info", choices=["debug", "info", "warn", "error"])
+
+    add_flags(sub.add_parser("full"))
+    add_flags(sub.add_parser("companies"))
+    add_flags(sub.add_parser("people"))
+
+    # global flags
+    ap.add_argument(
+        "--debug",
+        action="store_true",
+        help="Use built-in demo defaults (same as C4AI_DEMO_DEBUG=1)",
+    )
+    return ap
+
+
+def detect_debug_defaults(force = False) -> SimpleNamespace:
+    if not force and sys.gettrace() is None and not os.getenv("C4AI_DEMO_DEBUG"):
+        return SimpleNamespace()
+    # ----- debug‑friendly defaults -----
+    return SimpleNamespace(
+        cmd="full",
+        query="health insurance management",
+        geo=102713980,
+        # title_filters="Product,Engineering",
+        title_filters="",
+        max_companies=10,
+        max_people=5,
+        profile_name="profile_linkedin_uc",
+        outdir="./debug_out",
+        concurrency=2,
+        log_level="debug",
+    )
+
+
+async def async_main(opts):
+    # ─────────── logging setup ───────────
+    console = Console()
+    logging.basicConfig(
+        level=opts.log_level.upper(),
+        format="%(message)s",
+        handlers=[RichHandler(console=console, markup=True, rich_tracebacks=True)],
+    )
+
+    # -------------------------------------------------------------------
+    # Load or build schemas (one‑time LLM call each)
+    # -------------------------------------------------------------------
+    company_schema = _load_or_build_schema(
+        COMPANY_SCHEMA_PATH,
+        _SAMPLE_COMPANY_HTML,
+        _COMPANY_SCHEMA_QUERY,
+        _COMPANY_SCHEMA_EXAMPLE,
+        # True
+    )
+    people_schema = _load_or_build_schema(
+        PEOPLE_SCHEMA_PATH,
+        _SAMPLE_PEOPLE_HTML,
+        _PEOPLE_SCHEMA_QUERY,
+        _PEOPLE_SCHEMA_EXAMPLE,
+        # True
+    )
+
+    outdir = BASE_DIR / pathlib.Path(opts.outdir)
+    outdir.mkdir(parents=True, exist_ok=True)
+    f_companies = (BASE_DIR / outdir / "companies.jsonl").open("a", encoding="utf-8")
+    f_people = (BASE_DIR / outdir / "people.jsonl").open("a", encoding="utf-8")
+
+    # -------------------------------------------------------------------
+    # Prepare crawler with cookie pool rotation
+    # -------------------------------------------------------------------
+    profiler = BrowserProfiler()
+    path = profiler.get_profile_path(opts.profile_name)
+    bc = BrowserConfig(
+        headless=False,        
+        verbose=False,
+        user_data_dir=path,
+        use_managed_browser=True,
+        user_agent_mode = "random",
+        user_agent_generator_config= {
+            "platforms": "mobile",
+            "os": "Android"
+        }
+    )
+    crawler = AsyncWebCrawler(config=bc)
+    
+    await crawler.start()
+
+    # Single worker for simplicity; concurrency can be scaled by arun_many if needed.
+    # crawler = await next_crawler().start()
+    try:
+        # Build LinkedIn search URL
+        search_url = f'https://www.linkedin.com/search/results/companies/?keywords={quote(opts.query)}&companyHqGeo="{opts.geo}"'
+        logging.info("Seed URL => %s", search_url)
+
+        companies: List[Dict] = []
+        if opts.cmd in ("companies", "full"):
+            companies = await crawl_company_search(
+                crawler, search_url, company_schema, opts.max_companies
+            )
+            for c in companies:
+                f_companies.write(json.dumps(c, ensure_ascii=False) + "\n")
+            logging.info(f"[bold green]✓[/] Companies scraped so far: {len(companies)}")
+
+        if opts.cmd in ("people", "full"):
+            if not companies:
+                # load from previous run
+                src = outdir / "companies.jsonl"
+                if not src.exists():
+                    logging.error("companies.jsonl missing — run companies/full first")
+                    return 10
+                companies = [json.loads(l) for l in src.read_text().splitlines()]
+            total_people = 0
+            title_kw = " ".join([t.strip() for t in opts.title_filters.split(",") if t.strip()]) if opts.title_filters else ""
+            for comp in companies:
+                people = await crawl_people_page(
+                    crawler,
+                    comp["people_url"],
+                    people_schema,
+                    opts.max_people,
+                    title_kw,
+                )
+                for p in people:
+                    rec = p | {
+                        "company_handle": comp["handle"],
+                        # "captured_at": datetime.now(UTC).isoformat(timespec="seconds") + "Z",
+                        "captured_at": datetime.now(UTC).isoformat(timespec="seconds") + "Z",
+                    }
+                    f_people.write(json.dumps(rec, ensure_ascii=False) + "\n")
+                total_people += len(people)
+                logging.info(
+                    f"{comp['name']} — [cyan]{len(people)}[/] people extracted"
+                )
+                await asyncio.sleep(random.uniform(0.5, 1))
+            logging.info("Total people scraped: %d", total_people)
+    finally:
+        await crawler.close()
+        f_companies.close()
+        f_people.close()
+
+    return 0
+
+
+def main():
+    parser = build_arg_parser()
+    cli_opts = parser.parse_args()
+
+    # decide on debug defaults
+    if cli_opts.debug:
+        opts = detect_debug_defaults(force=True)
+        cli_opts = opts
+    else:
+        env_defaults = detect_debug_defaults()
+        opts = env_defaults if env_defaults else cli_opts
+
+    if not getattr(opts, "cmd", None):
+        opts.cmd = "full"
+
+    exit_code = asyncio.run(async_main(cli_opts))
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()