feat(pipeline): add high-level Crawler utility class for simplified web crawling

Add new Crawler class that provides a simplified interface for both single and batch URL crawling operations. Key features include:
- Simple single URL crawling with configurable options
- Parallel batch crawling with concurrency control
- Shared browser hub support for resource efficiency
- Progress tracking and custom retry strategies
- Comprehensive error handling and retry logic

Remove demo and extended test files in favor of new focused test suite.

.gitattributes

@@ -0,0 +1,12 @@
+# Documentation
+*.html linguist-documentation
+docs/* linguist-documentation
+docs/examples/* linguist-documentation
+docs/md_v2/* linguist-documentation
+
+# Explicitly mark Python as the main language
+*.py linguist-detectable=true
+*.py linguist-language=Python
+
+# Exclude HTML from language statistics
+*.html linguist-detectable=false

.github/DISCUSSION_TEMPLATE/feature-requests.yml

@@ -0,0 +1,59 @@
+title: "[Feature Request]: "
+labels: ["⚙️ New"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for your interest in suggesting a new feature! Before you submit, please take a moment to check if already exists in
+        this discussions category to avoid duplicates. 😊
+
+  - type: textarea
+    id: needs_to_be_done
+    attributes:
+      label: What needs to be done?
+      description: Please describe the feature or functionality you'd like to see.
+      placeholder: "e.g., Return alt text along with images scraped from a webpages in Result"
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem_to_solve
+    attributes:
+      label: What problem does this solve?
+      description: Explain the pain point or issue this feature will help address.
+      placeholder: "e.g., Bypass Captchas added by cloudflare"
+    validations:
+      required: true
+
+  - type: textarea
+    id: target_users
+    attributes:
+      label: Target users/beneficiaries
+      description: Who would benefit from this feature? (e.g., specific teams, developers, users, etc.)
+      placeholder: "e.g., Marketing teams, developers"
+    validations:
+      required: false
+
+  - type: textarea
+    id: current_workarounds
+    attributes:
+      label: Current alternatives/workarounds
+      description: Are there any existing solutions or workarounds? How does this feature improve upon them?
+      placeholder: "e.g., Users manually select the css classes mapped to data fields to extract them"
+    validations:
+      required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        ### 💡 Implementation Ideas
+
+  - type: textarea
+    id: proposed_approach
+    attributes:
+      label: Proposed approach
+      description: Share any ideas you have for how this feature could be implemented. Point out any challenges your foresee
+       and the success metrics for this feature
+      placeholder: "e.g., Implement a breadth first traversal algorithm for scraper"
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,127 @@
+name: Bug Report
+description: Report a bug with the Crawl4AI.
+title: "[Bug]: "
+labels: ["🐞 Bug","🩺 Needs Triage"]
+body:
+  - type: input
+    id: crawl4ai_version
+    attributes:
+      label: crawl4ai version
+      description: Specify the version of crawl4ai you are using.
+      placeholder: "e.g., 2.0.0"
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected_behavior
+    attributes:
+      label: Expected Behavior
+      description: Describe what you expected to happen.
+      placeholder: "Provide a detailed explanation of the expected outcome."
+    validations:
+      required: true
+
+  - type: textarea
+    id: current_behavior
+    attributes:
+      label: Current Behavior
+      description: Describe what is happening instead of the expected behavior.
+      placeholder: "Describe the actual result or issue you encountered."
+    validations:
+      required: true
+
+  - type: dropdown
+    id: reproducible
+    attributes:
+      label: Is this reproducible?
+      description: Indicate whether this bug can be reproduced consistently.
+      options:
+        - "Yes"
+        - "No"
+    validations:
+      required: true
+
+  - type: textarea
+    id: inputs
+    attributes:
+      label: Inputs Causing the Bug
+      description: Provide details about the inputs causing the issue.
+      placeholder: |
+        - URL(s): 
+        - Settings used: 
+        - Input data (if applicable): 
+      render: bash
+  
+  - type: textarea
+    id: steps_to_reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: Provide step-by-step instructions to reproduce the issue.
+      placeholder: |
+        1. Go to...
+        2. Click on...
+        3. Observe the issue...
+      render: bash
+  
+  - type: textarea
+    id: code_snippets
+    attributes:
+      label: Code snippets
+      description: Provide code snippets(if any). Add comments as necessary
+      placeholder: print("Hello world")
+      render: python
+
+  # Header Section with Title
+  - type: markdown
+    attributes:
+      value: |
+          ## Supporting Information
+          Please provide the following details to help us understand and resolve your issue. This will assist us in reproducing and diagnosing the problem
+
+  - type: input
+    id: os
+    attributes:
+      label: OS
+      description: Please provide the operating system & distro where the issue occurs.
+      placeholder: "e.g., Windows, macOS, Linux"
+    validations:
+      required: true
+
+  - type: input
+    id: python_version
+    attributes:
+      label: Python version
+      description: Specify the Python version being used.
+      placeholder: "e.g., 3.8.5"
+    validations:
+      required: true
+
+  # Browser Field
+  - type: input
+    id: browser
+    attributes:
+      label: Browser
+      description: Provide the name of the browser you are using.
+      placeholder: "e.g., Chrome, Firefox, Safari"
+    validations:
+      required: false
+
+  # Browser Version Field
+  - type: input
+    id: browser_version
+    attributes:
+      label: Browser version
+      description: Provide the version of the browser you are using.
+      placeholder: "e.g., 91.0.4472.124"
+    validations:
+      required: false
+
+  # Error Logs Field (Text Area)
+  - type: textarea
+    id: error_logs
+    attributes:
+      label: Error logs & Screenshots (if applicable)
+      description: If you encountered any errors, please provide the error logs. Attach any relevant screenshots to help us understand the issue.
+      placeholder: "Paste error logs here and attach your screenshots"
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Feature Requests
+    url: https://github.com/unclecode/crawl4ai/discussions/categories/feature-requests
+    about: "Suggest new features or enhancements for Crawl4AI"
+  - name: Forums - Q&A
+    url: https://github.com/unclecode/crawl4ai/discussions/categories/forums-q-a
+    about: "Ask questions or engage in general discussions about Crawl4AI"

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+## Summary
+Please include a summary of the change and/or which issues are fixed.
+
+eg: `Fixes #123` (Tag GitHub issue numbers in this format, so it automatically links the issues with your PR)
+
+## List of files changed and why
+eg: quickstart.py - To update the example as per new changes
+
+## How Has This Been Tested?
+Please describe the tests that you ran to verify your changes.
+
+## Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] I have added/updated unit tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes

.gitignore

@@ -165,6 +165,8 @@ Crawl4AI.egg-info/
 Crawl4AI.egg-info/*
 crawler_data.db
 .vscode/
+.tests/
+.test_pads/
 test_pad.py
 test_pad*.py
 .data/
@@ -187,4 +189,72 @@ a.txt
 .lambda_function.py
 ec2*
 
-update_changelog.sh
+update_changelog.sh
+
+.DS_Store
+docs/.DS_Store
+tmp/
+test_env/
+**/.DS_Store
+**/.DS_Store
+
+todo.md
+todo_executor.md
+git_changes.py
+git_changes.md
+pypi_build.sh
+git_issues.py
+git_issues.md
+
+.next/
+.tests/
+# .issues/
+.docs/
+.issues/
+.gitboss/
+todo_executor.md
+protect-all-except-feature.sh
+manage-collab.sh
+publish.sh
+combine.sh
+combined_output.txt
+.local
+.scripts
+tree.md
+tree.md
+.scripts
+.local
+.do
+/plans
+plans/
+
+# Codeium
+.codeiumignore
+todo/
+
+# Continue development files
+.continue/
+.continuerc.json
+continue.lock
+continue_core.log
+contextProviders/
+continue_workspace/
+.continue-cache/
+continue_config.json
+
+# Continue temporary files
+.continue-temp/
+.continue-logs/
+.continue-downloads/
+
+# Continue VS Code specific
+.vscode-continue/
+.vscode-continue-cache/
+
+.prompts/
+
+.llm.env
+.private/
+
+CLAUDE_MONITOR.md
+CLAUDE.md

CODE_OF_CONDUCT.md

@@ -0,0 +1,131 @@
+# Crawl4AI Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+unclecode@crawl4ai.com. All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTORS.md

@@ -0,0 +1,55 @@
+# Contributors to Crawl4AI
+
+We would like to thank the following people for their contributions to Crawl4AI:
+
+## Core Team
+
+- [Unclecode](https://github.com/unclecode) - Project Creator and Main Developer
+- [Nasrin](https://github.com/ntohidi) - Project Manager and Developer
+- [Aravind Karnam](https://github.com/aravindkarnam) - Head of Community and Product 
+
+## Community Contributors
+
+- [aadityakanjolia4](https://github.com/aadityakanjolia4) - Fix for `CustomHTML2Text` is not defined.
+- [FractalMind](https://github.com/FractalMind) - Created the first official Docker Hub image and fixed Dockerfile errors
+- [ketonkss4](https://github.com/ketonkss4) - Identified Selenium's new capabilities, helping reduce dependencies
+- [jonymusky](https://github.com/jonymusky) - Javascript execution documentation, and wait_for
+- [datehoer](https://github.com/datehoer) - Add browser prxy support
+
+## Pull Requests
+
+- [dvschuyl](https://github.com/dvschuyl) - AsyncPlaywrightCrawlerStrategy page-evaluate context destroyed by navigation [#304](https://github.com/unclecode/crawl4ai/pull/304)
+- [nelzomal](https://github.com/nelzomal) - Enhance development installation instructions [#286](https://github.com/unclecode/crawl4ai/pull/286)
+- [HamzaFarhan](https://github.com/HamzaFarhan) - Handled the cases where markdown_with_citations, references_markdown, and filtered_html might not be defined [#293](https://github.com/unclecode/crawl4ai/pull/293)
+- [NanmiCoder](https://github.com/NanmiCoder) - fix: crawler strategy exception handling and fixes [#271](https://github.com/unclecode/crawl4ai/pull/271)
+- [paulokuong](https://github.com/paulokuong) - fix: RAWL4_AI_BASE_DIRECTORY should be Path object instead of string [#298](https://github.com/unclecode/crawl4ai/pull/298)
+
+#### Feb-Alpha-1
+- [sufianuddin](https://github.com/sufianuddin) - fix: [Documentation for JsonCssExtractionStrategy](https://github.com/unclecode/crawl4ai/issues/651)
+- [tautikAg](https://github.com/tautikAg) - fix: [Markdown output has incorect spacing](https://github.com/unclecode/crawl4ai/issues/599)
+- [cardit1](https://github.com/cardit1) - fix: ['AsyncPlaywrightCrawlerStrategy' object has no attribute 'downloads_path'](https://github.com/unclecode/crawl4ai/issues/585)
+- [dmurat](https://github.com/dmurat) - fix: [ Incorrect rendering of inline code inside of links ](https://github.com/unclecode/crawl4ai/issues/583)
+- [Sparshsing](https://github.com/Sparshsing) - fix: [Relative Urls in the webpage not extracted properly ](https://github.com/unclecode/crawl4ai/issues/570)
+
+
+
+## Other Contributors
+
+- [Gokhan](https://github.com/gkhngyk) 
+- [Shiv Kumar](https://github.com/shivkumar0757)
+- [QIN2DIM](https://github.com/QIN2DIM)
+
+#### Typo fixes
+- [ssoydan](https://github.com/ssoydan)
+- [Darshan](https://github.com/Darshan2104)
+- [tuhinmallick](https://github.com/tuhinmallick)
+
+## Acknowledgements
+
+We also want to thank all the users who have reported bugs, suggested features, or helped in any other way to make Crawl4AI better.
+
+---
+
+If you've contributed to Crawl4AI and your name isn't on this list, please [open a pull request](https://github.com/unclecode/crawl4ai/pulls) with your name, link, and contribution, and we'll review it promptly.
+
+Thank you all for your contributions!

Dockerfile

@@ -1,58 +1,156 @@
-# First stage: Build and install dependencies
-FROM python:3.10-slim-bookworm
+FROM python:3.10-slim
 
-# Set the working directory in the container
-WORKDIR /usr/src/app
+# Set build arguments
+ARG APP_HOME=/app
+ARG GITHUB_REPO=https://github.com/unclecode/crawl4ai.git
+ARG GITHUB_BRANCH=main
+ARG USE_LOCAL=true
 
-# Install build dependencies
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    wget \
-    git \
+ENV PYTHONFAULTHANDLER=1 \
+    PYTHONHASHSEED=random \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_DEFAULT_TIMEOUT=100 \
+    DEBIAN_FRONTEND=noninteractive \
+    REDIS_HOST=localhost \
+    REDIS_PORT=6379
+
+ARG PYTHON_VERSION=3.10
+ARG INSTALL_TYPE=default
+ARG ENABLE_GPU=false
+ARG TARGETARCH
+
+LABEL maintainer="unclecode"
+LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
+LABEL version="1.0"    
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
     curl \
-    unzip \
+    wget \
     gnupg \
-    xvfb \
-    ca-certificates \
-    apt-transport-https \
-    software-properties-common && \
-    rm -rf /var/lib/apt/lists/*    
+    git \
+    cmake \
+    pkg-config \
+    python3-dev \
+    libjpeg-dev \
+    redis-server \
+    supervisor \
+    && rm -rf /var/lib/apt/lists/*
 
-# Copy the application code
-COPY . .
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libglib2.0-0 \
+    libnss3 \
+    libnspr4 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdrm2 \
+    libdbus-1-3 \
+    libxcb1 \
+    libxkbcommon0 \
+    libx11-6 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxrandr2 \
+    libgbm1 \
+    libpango-1.0-0 \
+    libcairo2 \
+    libasound2 \
+    libatspi2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install Crawl4AI using the local setup.py (which will use the default installation)
-RUN pip install --no-cache-dir .
+RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \
+    apt-get update && apt-get install -y --no-install-recommends \
+    nvidia-cuda-toolkit \
+    && rm -rf /var/lib/apt/lists/* ; \
+else \
+    echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
+fi
 
-# Install Google Chrome and ChromeDriver
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
-    sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list' && \
-    apt-get update && \
-    apt-get install -y google-chrome-stable && \
-    wget -O /tmp/chromedriver.zip http://chromedriver.storage.googleapis.com/`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE`/chromedriver_linux64.zip && \
-    unzip /tmp/chromedriver.zip chromedriver -d /usr/local/bin/
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+    echo "🦾 Installing ARM-specific optimizations"; \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libopenblas-dev \
+    && rm -rf /var/lib/apt/lists/*; \
+elif [ "$TARGETARCH" = "amd64" ]; then \
+    echo "🖥️ Installing AMD64-specific optimizations"; \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libomp-dev \
+    && rm -rf /var/lib/apt/lists/*; \
+else \
+    echo "Skipping platform-specific optimizations (unsupported platform)"; \
+fi
 
-# Set environment to use Chrome and ChromeDriver properly
-ENV CHROME_BIN=/usr/bin/google-chrome \
-    CHROMEDRIVER=/usr/local/bin/chromedriver \
-    DISPLAY=:99 \
-    DBUS_SESSION_BUS_ADDRESS=/dev/null \
-    PYTHONUNBUFFERED=1
+WORKDIR ${APP_HOME}
 
-# Ensure the PATH environment variable includes the location of the installed packages
-ENV PATH /opt/conda/bin:$PATH   
+RUN echo '#!/bin/bash\n\
+if [ "$USE_LOCAL" = "true" ]; then\n\
+    echo "📦 Installing from local source..."\n\
+    pip install --no-cache-dir /tmp/project/\n\
+else\n\
+    echo "🌐 Installing from GitHub..."\n\
+    for i in {1..3}; do \n\
+        git clone --branch ${GITHUB_BRANCH} ${GITHUB_REPO} /tmp/crawl4ai && break || \n\
+        { echo "Attempt $i/3 failed! Taking a short break... ☕"; sleep 5; }; \n\
+    done\n\
+    pip install --no-cache-dir /tmp/crawl4ai\n\
+fi' > /tmp/install.sh && chmod +x /tmp/install.sh
 
-# Make port 80 available to the world outside this container
-EXPOSE 80
+COPY . /tmp/project/
 
-# Download models call cli "crawl4ai-download-models"
-# RUN crawl4ai-download-models
+COPY deploy/docker/supervisord.conf .
 
-# Install mkdocs
-RUN pip install mkdocs mkdocs-terminal
+COPY deploy/docker/requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
 
-# Call mkdocs to build the documentation
-RUN mkdocs build
+RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
+        pip install --no-cache-dir \
+            torch \
+            torchvision \
+            torchaudio \
+            scikit-learn \
+            nltk \
+            transformers \
+            tokenizers && \
+        python -m nltk.downloader punkt stopwords ; \
+    fi
 
-# Run uvicorn
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "80", "--workers", "4"]
+RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
+        pip install "/tmp/project/[all]" && \
+        python -m crawl4ai.model_loader ; \
+    elif [ "$INSTALL_TYPE" = "torch" ] ; then \
+        pip install "/tmp/project/[torch]" ; \
+    elif [ "$INSTALL_TYPE" = "transformer" ] ; then \
+        pip install "/tmp/project/[transformer]" && \
+        python -m crawl4ai.model_loader ; \
+    else \
+        pip install "/tmp/project" ; \
+    fi
+    
+RUN pip install --no-cache-dir --upgrade pip && \
+    /tmp/install.sh && \
+    python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
+    python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')"
+    
+RUN playwright install --with-deps chromium
+
+COPY deploy/docker/* ${APP_HOME}/
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD bash -c '\
+    MEM=$(free -m | awk "/^Mem:/{print \$2}"); \
+    if [ $MEM -lt 2048 ]; then \
+        echo "⚠️ Warning: Less than 2GB RAM available! Your container might need a memory boost! 🚀"; \
+        exit 1; \
+    fi && \
+    redis-cli ping > /dev/null && \
+    curl -f http://localhost:8000/health || exit 1'
+
+EXPOSE 6379
+CMD ["supervisord", "-c", "supervisord.conf"]
+    

Dockerfile_mac

@@ -1,44 +0,0 @@
-# Use an official Python runtime as a parent image
-FROM python:3.10-slim
-
-# Set the working directory in the container
-WORKDIR /usr/src/app
-
-# Copy the current directory contents into the container at /usr/src/app
-COPY . .
-
-# Install any needed packages specified in requirements.txt
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Install dependencies for Chrome and ChromeDriver
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    wget \
-    xvfb \
-    unzip \
-    curl \
-    gnupg2 \
-    ca-certificates \
-    apt-transport-https \
-    software-properties-common \
-    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
-    && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
-    && apt-get update \
-    && apt-get install -y google-chrome-stable \
-    && rm -rf /var/lib/apt/lists/* \
-    && apt install chromium-chromedriver -y
-
-# Install spacy library using pip
-RUN pip install spacy
-
-# Set display port and dbus env to avoid hanging
-ENV DISPLAY=:99
-ENV DBUS_SESSION_BUS_ADDRESS=/dev/null
-
-# Make port 80 available to the world outside this container
-EXPOSE 80
-
-# Define environment variable
-ENV PYTHONUNBUFFERED 1
-
-# Run uvicorn
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "80", "--workers", "4"]

LICENSE

@@ -48,4 +48,22 @@ You may add Your own copyright statement to Your modifications and may provide a
 
 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
 
-END OF TERMS AND CONDITIONS
+END OF TERMS AND CONDITIONS
+
+---
+Attribution Requirement
+
+All distributions, publications, or public uses of this software, or derivative works based on this software, must include the following attribution:
+
+"This product includes software developed by UncleCode (https://x.com/unclecode) as part of the Crawl4AI project (https://github.com/unclecode/crawl4ai)."
+
+This attribution must be displayed in a prominent and easily accessible location, such as:
+
+-   For software distributions: In a NOTICE file, README file, or equivalent documentation.
+-   For publications (research papers, articles, blog posts): In the acknowledgments section or a footnote.
+-   For websites/web applications: In an "About" or "Credits" section.
+-   For command-line tools: In the help/usage output.
+
+This requirement ensures proper credit is given for the use of Crawl4AI and helps promote the project.
+
+---

MANIFEST.in

@@ -0,0 +1,2 @@
+include requirements.txt
+recursive-include crawl4ai/js_snippet *.js

MISSION.md

@@ -0,0 +1,46 @@
+# Mission
+
+![Mission Diagram](./docs/assets/pitch-dark.svg)
+
+### 1. The Data Capitalization Opportunity
+
+We live in an unprecedented era of digital wealth creation. Every day, individuals and enterprises generate massive amounts of valuable digital footprints across various platforms, social media channels, messenger apps, and cloud services. While people can interact with their data within these platforms, there's an immense untapped opportunity to transform this data into true capital assets. Just as physical property became a foundational element of wealth creation, personal and enterprise data has the potential to become a new form of capital on balance sheets.
+
+For individuals, this represents an opportunity to transform their digital activities into valuable assets. For enterprises, their internal communications, team discussions, and collaborative documents contain rich insights that could be structured and valued as intellectual capital. This wealth of information represents an unprecedented opportunity for value creation in the digital age.
+
+### 2. The Potential of Authentic Data
+
+While synthetic data has played a crucial role in AI development, there's an enormous untapped potential in the authentic data generated by individuals and organizations. Every message, document, and interaction contains unique insights and patterns that could enhance AI development. The challenge isn't a lack of data - it's that most authentic human-generated data remains inaccessible for productive use.
+
+By enabling willing participation in data sharing, we can unlock this vast reservoir of authentic human knowledge. This represents an opportunity to enhance AI development with diverse, real-world data that reflects the full spectrum of human experience and knowledge.
+
+## Our Pathway to Data Democracy
+
+### 1. Open-Source Foundation
+
+Our first step is creating an open-source data extraction engine that empowers developers and innovators to build tools for data structuring and organization. This foundation ensures transparency, security, and community-driven development. By making these tools openly available, we enable the technical infrastructure needed for true data ownership and capitalization.
+
+### 2. Data Capitalization Platform
+
+Building on this open-source foundation, we're developing a platform that helps individuals and enterprises transform their digital footprints into structured, valuable assets. This platform will provide the tools and frameworks needed to organize, understand, and value personal and organizational data as true capital assets.
+
+### 3. Creating a Data Marketplace
+
+The final piece is establishing a marketplace where individuals and organizations can willingly share their data assets. This creates opportunities for:
+- Individuals to earn equity, revenue, or other forms of value from their data
+- Enterprises to access diverse, high-quality data for AI development
+- Researchers to work with authentic human-generated data
+- Startups to build innovative solutions using real-world data
+
+## Economic Vision: A Shared Data Economy
+
+We envision a future where data becomes a fundamental asset class in a thriving shared economy. This transformation will democratize AI development by enabling willing participation in data sharing, ensuring that the benefits of AI advancement flow back to data creators. Just as property rights revolutionized economic systems, establishing data as a capital asset will create new opportunities for wealth creation and economic participation.
+
+This shared data economy will:
+- Enable individuals to capitalize on their digital footprints
+- Create new revenue streams for data creators
+- Provide AI developers with access to diverse, authentic data
+- Foster innovation through broader access to real-world data
+- Ensure more equitable distribution of AI's economic benefits
+
+Our vision is to facilitate this transformation from the ground up - starting with open-source tools, progressing to data capitalization platforms, and ultimately creating a thriving marketplace where data becomes a true asset class in a shared economy. This approach ensures that the future of AI is built on a foundation of authentic human knowledge, with benefits flowing back to the individuals and organizations who create and share their valuable data.

README.md

@@ -1,161 +1,681 @@
-# Crawl4AI v0.2.72 🕷️🤖
+# 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper.
+
+<div align="center">
+
+<a href="https://trendshift.io/repositories/11716" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11716" alt="unclecode%2Fcrawl4ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 
 [![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
 [![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members)
-[![GitHub Issues](https://img.shields.io/github/issues/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/issues)
-[![GitHub Pull Requests](https://img.shields.io/github/issues-pr/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/pulls)
-[![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)
 
-Crawl4AI simplifies web crawling and data extraction, making it accessible for large language models (LLMs) and AI applications. 🆓🌐
+[![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai)
+[![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai)](https://pypi.org/project/crawl4ai/)
+[![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
+
+<!-- [![Documentation Status](https://readthedocs.org/projects/crawl4ai/badge/?version=latest)](https://crawl4ai.readthedocs.io/) -->
+[![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md)
+
+</div>
+
+Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for LLMs, AI agents, and data pipelines. Open source, flexible, and built for real-time performance, Crawl4AI empowers developers with unmatched speed, precision, and deployment ease.  
+
+[✨ Check out latest update v0.5.0](#-recent-updates)
+
+🎉 **Version 0.5.0 is out!** This major release introduces Deep Crawling with BFS/DFS/BestFirst strategies, Memory-Adaptive Dispatcher, Multiple Crawling Strategies (Playwright and HTTP), Docker Deployment with FastAPI, Command-Line Interface (CLI), and more! [Read the release notes →](https://docs.crawl4ai.com/blog)
+
+<details>
+<summary>🤓 <strong>My Personal Story</strong></summary>
+
+My journey with computers started in childhood when my dad, a computer scientist, introduced me to an Amstrad computer. Those early days sparked a fascination with technology, leading me to pursue computer science and specialize in NLP during my postgraduate studies. It was during this time that I first delved into web crawling, building tools to help researchers organize papers and extract information from publications a challenging yet rewarding experience that honed my skills in data extraction.
+
+Fast forward to 2023, I was working on a tool for a project and needed a crawler to convert a webpage into markdown. While exploring solutions, I found one that claimed to be open-source but required creating an account and generating an API token. Worse, it turned out to be a SaaS model charging $16, and its quality didn’t meet my standards. Frustrated, I realized this was a deeper problem. That frustration turned into turbo anger mode, and I decided to build my own solution. In just a few days, I created Crawl4AI. To my surprise, it went viral, earning thousands of GitHub stars and resonating with a global community.
+
+I made Crawl4AI open-source for two reasons. First, it’s my way of giving back to the open-source community that has supported me throughout my career. Second, I believe data should be accessible to everyone, not locked behind paywalls or monopolized by a few. Open access to data lays the foundation for the democratization of AI, a vision where individuals can train their own models and take ownership of their information. This library is the first step in a larger journey to create the best open-source data extraction and generation tool the world has ever seen, built collaboratively by a passionate community.
+
+Thank you to everyone who has supported this project, used it, and shared feedback. Your encouragement motivates me to dream even bigger. Join us, file issues, submit PRs, or spread the word. Together, we can build a tool that truly empowers people to access their own data and reshape the future of AI.
+</details>
+
+## 🧐 Why Crawl4AI?
+
+1. **Built for LLMs**: Creates smart, concise Markdown optimized for RAG and fine-tuning applications.  
+2. **Lightning Fast**: Delivers results 6x faster with real-time, cost-efficient performance.  
+3. **Flexible Browser Control**: Offers session management, proxies, and custom hooks for seamless data access.  
+4. **Heuristic Intelligence**: Uses advanced algorithms for efficient extraction, reducing reliance on costly models.  
+5. **Open Source & Deployable**: Fully open-source with no API keys—ready for Docker and cloud integration.  
+6. **Thriving Community**: Actively maintained by a vibrant community and the #1 trending GitHub repository.
+
+## 🚀 Quick Start 
+
+1. Install Crawl4AI:
+```bash
+# Install the package
+pip install -U crawl4ai
+
+# For pre release versions
+pip install crawl4ai --pre
+
+# Run post-installation setup
+crawl4ai-setup
+
+# Verify your installation
+crawl4ai-doctor
+```
+
+If you encounter any browser-related issues, you can install them manually:
+```bash
+python -m playwright install --with-deps chromium
+```
+
+2. Run a simple web crawl with Python:
+```python
+import asyncio
+from crawl4ai import *
+
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+        )
+        print(result.markdown)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+3. Or use the new command-line interface:
+```bash
+# Basic crawl with markdown output
+crwl https://www.nbcnews.com/business -o markdown
+
+# Deep crawl with BFS strategy, max 10 pages
+crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10
+
+# Use LLM extraction with a specific question
+crwl https://www.example.com/products -q "Extract all product prices"
+```
+
+## ✨ Features 
+
+<details>
+<summary>📝 <strong>Markdown Generation</strong></summary>
+
+- 🧹 **Clean Markdown**: Generates clean, structured Markdown with accurate formatting.
+- 🎯 **Fit Markdown**: Heuristic-based filtering to remove noise and irrelevant parts for AI-friendly processing.
+- 🔗 **Citations and References**: Converts page links into a numbered reference list with clean citations.
+- 🛠️ **Custom Strategies**: Users can create their own Markdown generation strategies tailored to specific needs.
+- 📚 **BM25 Algorithm**: Employs BM25-based filtering for extracting core information and removing irrelevant content. 
+</details>
+
+<details>
+<summary>📊 <strong>Structured Data Extraction</strong></summary>
+
+- 🤖 **LLM-Driven Extraction**: Supports all LLMs (open-source and proprietary) for structured data extraction.
+- 🧱 **Chunking Strategies**: Implements chunking (topic-based, regex, sentence-level) for targeted content processing.
+- 🌌 **Cosine Similarity**: Find relevant content chunks based on user queries for semantic extraction.
+- 🔎 **CSS-Based Extraction**: Fast schema-based data extraction using XPath and CSS selectors.
+- 🔧 **Schema Definition**: Define custom schemas for extracting structured JSON from repetitive patterns.
+
+</details>
+
+<details>
+<summary>🌐 <strong>Browser Integration</strong></summary>
+
+- 🖥️ **Managed Browser**: Use user-owned browsers with full control, avoiding bot detection.
+- 🔄 **Remote Browser Control**: Connect to Chrome Developer Tools Protocol for remote, large-scale data extraction.
+- 👤 **Browser Profiler**: Create and manage persistent profiles with saved authentication states, cookies, and settings.
+- 🔒 **Session Management**: Preserve browser states and reuse them for multi-step crawling.
+- 🧩 **Proxy Support**: Seamlessly connect to proxies with authentication for secure access.
+- ⚙️ **Full Browser Control**: Modify headers, cookies, user agents, and more for tailored crawling setups.
+- 🌍 **Multi-Browser Support**: Compatible with Chromium, Firefox, and WebKit.
+- 📐 **Dynamic Viewport Adjustment**: Automatically adjusts the browser viewport to match page content, ensuring complete rendering and capturing of all elements.
+
+</details>
+
+<details>
+<summary>🔎 <strong>Crawling & Scraping</strong></summary>
+
+- 🖼️ **Media Support**: Extract images, audio, videos, and responsive image formats like `srcset` and `picture`.
+- 🚀 **Dynamic Crawling**: Execute JS and wait for async or sync for dynamic content extraction.
+- 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis.
+- 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`).
+- 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content.
+- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior.
+- 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches.
+- 📄 **Metadata Extraction**: Retrieve structured metadata from web pages.
+- 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content.
+- 🕵️ **Lazy Load Handling**: Waits for images to fully load, ensuring no content is missed due to lazy loading.
+- 🔄 **Full-Page Scanning**: Simulates scrolling to load and capture all dynamic content, perfect for infinite scroll pages.
+
+</details>
+
+<details>
+<summary>🚀 <strong>Deployment</strong></summary>
+
+- 🐳 **Dockerized Setup**: Optimized Docker image with FastAPI server for easy deployment.
+- 🔑 **Secure Authentication**: Built-in JWT token authentication for API security.
+- 🔄 **API Gateway**: One-click deployment with secure token authentication for API-based workflows.
+- 🌐 **Scalable Architecture**: Designed for mass-scale production and optimized server performance.
+- ☁️ **Cloud Deployment**: Ready-to-deploy configurations for major cloud platforms.
+
+</details>
+
+<details>
+<summary>🎯 <strong>Additional Features</strong></summary>
+
+- 🕶️ **Stealth Mode**: Avoid bot detection by mimicking real users.
+- 🏷️ **Tag-Based Content Extraction**: Refine crawling based on custom tags, headers, or metadata.
+- 🔗 **Link Analysis**: Extract and analyze all links for detailed data exploration.
+- 🛡️ **Error Handling**: Robust error management for seamless execution.
+- 🔐 **CORS & Static Serving**: Supports filesystem-based caching and cross-origin requests.
+- 📖 **Clear Documentation**: Simplified and updated guides for onboarding and advanced usage.
+- 🙌 **Community Recognition**: Acknowledges contributors and pull requests for transparency.
+
+</details>
 
 ## Try it Now!
 
-- Use as REST API: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1zODYjhemJ5bUmYceWpVoBMVpd0ofzNBZ?usp=sharing)
-- Use as Python library: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1wz8u30rvbq6Scodye9AGCw8Qg_Z8QGsk)
+✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing)
 
-✨ visit our [Documentation Website](https://crawl4ai.com/mkdocs/)
+✨ Visit our [Documentation Website](https://docs.crawl4ai.com/)
 
-## Features ✨
+## Installation 🛠️
 
-- 🆓 Completely free and open-source
-- 🤖 LLM-friendly output formats (JSON, cleaned HTML, markdown)
-- 🌍 Supports crawling multiple URLs simultaneously
-- 🎨 Extracts and returns all media tags (Images, Audio, and Video)
-- 🔗 Extracts all external and internal links
-- 📚 Extracts metadata from the page
-- 🔄 Custom hooks for authentication, headers, and page modifications before crawling
-- 🕵️ User-agent customization
-- 🖼️ Takes screenshots of the page
-- 📜 Executes multiple custom JavaScripts before crawling
-- 📚 Various chunking strategies: topic-based, regex, sentence, and more
-- 🧠 Advanced extraction strategies: cosine clustering, LLM, and more
-- 🎯 CSS selector support
-- 📝 Passes instructions/keywords to refine extraction
+Crawl4AI offers flexible installation options to suit various use cases. You can install it as a Python package or use Docker.
 
-## Cool Examples 🚀
+<details>
+<summary>🐍 <strong>Using pip</strong></summary>
 
-### Quick Start
+Choose the installation option that best fits your needs:
 
-```python
-from crawl4ai import WebCrawler
+### Basic Installation
 
-# Create an instance of WebCrawler
-crawler = WebCrawler()
-
-# Warm up the crawler (load necessary models)
-crawler.warmup()
-
-# Run the crawler on a URL
-result = crawler.run(url="https://www.nbcnews.com/business")
-
-# Print the extracted content
-print(result.markdown)
-```
-
-## How to install 🛠    
-```bash
-virtualenv venv
-source venv/bin/activate
-pip install "crawl4ai @ git+https://github.com/unclecode/crawl4ai.git"
-```️
-
-### Speed-First Design 🚀
-
-Perhaps the most important design principle for this library is speed. We need to ensure it can handle many links and resources in parallel as quickly as possible. By combining this speed with fast LLMs like Groq, the results will be truly amazing.
-
-```python
-import time
-from crawl4ai.web_crawler import WebCrawler
-crawler = WebCrawler()
-crawler.warmup()
-
-start = time.time()
-url = r"https://www.nbcnews.com/business"
-result = crawler.run( url, word_count_threshold=10, bypass_cache=True)
-end = time.time()
-print(f"Time taken: {end - start}")
-```
-
-Let's take a look the calculated time for the above code snippet:
+For basic web crawling and scraping tasks:
 
 ```bash
-[LOG] 🚀 Crawling done, success: True, time taken: 1.3623387813568115 seconds
-[LOG] 🚀 Content extracted, success: True, time taken: 0.05715131759643555 seconds
-[LOG] 🚀 Extraction, time taken: 0.05750393867492676 seconds.
-Time taken: 1.439958095550537
+pip install crawl4ai
+crawl4ai-setup # Setup the browser
 ```
-Fetching the content from the page took 1.3623 seconds, and extracting the content took 0.0575 seconds. 🚀
 
-### Extract Structured Data from Web Pages 📊
+By default, this will install the asynchronous version of Crawl4AI, using Playwright for web crawling.
 
-Crawl all OpenAI models and their fees from the official page.
+👉 **Note**: When you install Crawl4AI, the `crawl4ai-setup` should automatically install and set up Playwright. However, if you encounter any Playwright-related errors, you can manually install it using one of these methods:
+
+1. Through the command line:
+
+   ```bash
+   playwright install
+   ```
+
+2. If the above doesn't work, try this more specific command:
+
+   ```bash
+   python -m playwright install chromium
+   ```
+
+This second method has proven to be more reliable in some cases.
+
+---
+
+### Installation with Synchronous Version
+
+The sync version is deprecated and will be removed in future versions. If you need the synchronous version using Selenium:
+
+```bash
+pip install crawl4ai[sync]
+```
+
+---
+
+### Development Installation
+
+For contributors who plan to modify the source code:
+
+```bash
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+pip install -e .                    # Basic installation in editable mode
+```
+
+Install optional features:
+
+```bash
+pip install -e ".[torch]"           # With PyTorch features
+pip install -e ".[transformer]"     # With Transformer features
+pip install -e ".[cosine]"          # With cosine similarity features
+pip install -e ".[sync]"            # With synchronous crawling (Selenium)
+pip install -e ".[all]"             # Install all optional features
+```
+
+</details>
+
+<details>
+<summary>🐳 <strong>Docker Deployment</strong></summary>
+
+> 🚀 **Major Changes Coming!** We're developing a completely new Docker implementation that will make deployment even more efficient and seamless. The current Docker setup is being deprecated in favor of this new solution.
+
+### Current Docker Support
+
+The existing Docker implementation is being deprecated and will be replaced soon. If you still need to use Docker with the current version:
+
+- 📚 [Deprecated Docker Setup](./docs/deprecated/docker-deployment.md) - Instructions for the current Docker implementation
+- ⚠️ Note: This setup will be replaced in the next major release
+
+### What's Coming Next?
+
+Our new Docker implementation will bring:
+- Improved performance and resource efficiency
+- Streamlined deployment process
+- Better integration with Crawl4AI features
+- Enhanced scalability options
+
+Stay connected with our [GitHub repository](https://github.com/unclecode/crawl4ai) for updates!
+
+</details>
+
+---
+
+### Quick Test
+
+Run a quick test (works for both Docker options):
+
+```python
+import requests
+
+# Submit a crawl job
+response = requests.post(
+    "http://localhost:11235/crawl",
+    json={"urls": "https://example.com", "priority": 10}
+)
+task_id = response.json()["task_id"]
+
+# Continue polling until the task is complete (status="completed")
+result = requests.get(f"http://localhost:11235/task/{task_id}")
+```
+
+For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://docs.crawl4ai.com/basic/docker-deployment/).
+
+</details>
+
+
+## 🔬 Advanced Usage Examples 🔬
+
+You can check the project structure in the directory [https://github.com/unclecode/crawl4ai/docs/examples](docs/examples). Over there, you can find a variety of examples; here, some popular examples are shared.
+
+<details>
+<summary>📝 <strong>Heuristic Markdown Generation with Clean and Fit Markdown</strong></summary>
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def main():
+    browser_config = BrowserConfig(
+        headless=True,  
+        verbose=True,
+    )
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.ENABLED,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed", min_word_threshold=0)
+        ),
+        # markdown_generator=DefaultMarkdownGenerator(
+        #     content_filter=BM25ContentFilter(user_query="WHEN_WE_FOCUS_BASED_ON_A_USER_QUERY", bm25_threshold=1.0)
+        # ),
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url="https://docs.micronaut.io/4.7.6/guide/",
+            config=run_config
+        )
+        print(len(result.markdown.raw_markdown))
+        print(len(result.markdown.fit_markdown))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>🖥️ <strong>Executing JavaScript & Extract Structured Data without LLMs</strong></summary>
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+import json
+
+async def main():
+    schema = {
+    "name": "KidoCode Courses",
+    "baseSelector": "section.charge-methodology .w-tab-content > div",
+    "fields": [
+        {
+            "name": "section_title",
+            "selector": "h3.heading-50",
+            "type": "text",
+        },
+        {
+            "name": "section_description",
+            "selector": ".charge-content",
+            "type": "text",
+        },
+        {
+            "name": "course_name",
+            "selector": ".text-block-93",
+            "type": "text",
+        },
+        {
+            "name": "course_description",
+            "selector": ".course-content-text",
+            "type": "text",
+        },
+        {
+            "name": "course_icon",
+            "selector": ".image-92",
+            "type": "attribute",
+            "attribute": "src"
+        }
+    }
+}
+
+    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+    browser_config = BrowserConfig(
+        headless=False,
+        verbose=True
+    )
+    run_config = CrawlerRunConfig(
+        extraction_strategy=extraction_strategy,
+        js_code=["""(async () => {const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");for(let tab of tabs) {tab.scrollIntoView();tab.click();await new Promise(r => setTimeout(r, 500));}})();"""],
+        cache_mode=CacheMode.BYPASS
+    )
+        
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        
+        result = await crawler.arun(
+            url="https://www.kidocode.com/degrees/technology",
+            config=run_config
+        )
+
+        companies = json.loads(result.extracted_content)
+        print(f"Successfully extracted {len(companies)} companies")
+        print(json.dumps(companies[0], indent=2))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</details>
+
+<details>
+<summary>📚 <strong>Extracting Structured Data with LLMs</strong></summary>
 
 ```python
 import os
-from crawl4ai import WebCrawler
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
 from crawl4ai.extraction_strategy import LLMExtractionStrategy
 from pydantic import BaseModel, Field
 
 class OpenAIModelFee(BaseModel):
     model_name: str = Field(..., description="Name of the OpenAI model.")
     input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
-    output_fee: str = Field(..., description="Fee for output token ßfor the OpenAI model.")
+    output_fee: str = Field(..., description="Fee for output token for the OpenAI model.")
 
-url = 'https://openai.com/api/pricing/'
-crawler = WebCrawler()
-crawler.warmup()
-
-result = crawler.run(
-        url=url,
+async def main():
+    browser_config = BrowserConfig(verbose=True)
+    run_config = CrawlerRunConfig(
         word_count_threshold=1,
-        extraction_strategy= LLMExtractionStrategy(
-            provider= "openai/gpt-4o", api_token = os.getenv('OPENAI_API_KEY'), 
+        extraction_strategy=LLMExtractionStrategy(
+            # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2
+            # provider="ollama/qwen2", api_token="no-token", 
+            llm_config = LLMConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')), 
             schema=OpenAIModelFee.schema(),
             extraction_type="schema",
             instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
             Do not miss any models in the entire content. One extracted model JSON format should look like this: 
             {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""
         ),            
-        bypass_cache=True,
+        cache_mode=CacheMode.BYPASS,
     )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url='https://openai.com/api/pricing/',
+            config=run_config
+        )
+        print(result.extracted_content)
 
-print(result.extracted_content)
+if __name__ == "__main__":
+    asyncio.run(main())
 ```
 
-### Execute JS, Filter Data with CSS Selector, and Clustering
+</details>
+
+<details>
+<summary>🤖 <strong>Using You own Browser with Custom User Profile</strong></summary>
 
 ```python
-from crawl4ai import WebCrawler
-from crawl4ai.chunking_strategy import CosineStrategy
+import os, sys
+from pathlib import Path
+import asyncio, time
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
 
-js_code = ["const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"]
+async def test_news_crawl():
+    # Create a persistent user data directory
+    user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
+    os.makedirs(user_data_dir, exist_ok=True)
 
-crawler = WebCrawler()
-crawler.warmup()
-
-result = crawler.run(
-    url="https://www.nbcnews.com/business",
-    js=js_code,
-    css_selector="p",
-    extraction_strategy=CosineStrategy(semantic_filter="technology")
-)
-
-print(result.extracted_content)
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=True,
+        user_data_dir=user_data_dir,
+        use_persistent_context=True,
+    )
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        url = "ADDRESS_OF_A_CHALLENGING_WEBSITE"
+        
+        result = await crawler.arun(
+            url,
+            config=run_config,
+            magic=True,
+        )
+        
+        print(f"Successfully crawled {url}")
+        print(f"Content length: {len(result.markdown)}")
 ```
 
-## Documentation 📚
+</details>
 
-For detailed documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://crawl4ai.com/mkdocs/).
+## ✨ Recent Updates
 
-## Contributing 🤝
+### Version 0.5.0 Major Release Highlights
 
-We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTING.md) for more information.
+-   **🚀 Deep Crawling System**: Explore websites beyond initial URLs with three strategies:
+    -   **BFS Strategy**: Breadth-first search explores websites level by level
+    -   **DFS Strategy**: Depth-first search explores each branch deeply before backtracking
+    -   **BestFirst Strategy**: Uses scoring functions to prioritize which URLs to crawl next
+    -   **Page Limiting**: Control the maximum number of pages to crawl with `max_pages` parameter
+    -   **Score Thresholds**: Filter URLs based on relevance scores
+-   **⚡ Memory-Adaptive Dispatcher**: Dynamically adjusts concurrency based on system memory with built-in rate limiting
+-   **🔄 Multiple Crawling Strategies**:
+    -   **AsyncPlaywrightCrawlerStrategy**: Browser-based crawling with JavaScript support (Default)
+    -   **AsyncHTTPCrawlerStrategy**: Fast, lightweight HTTP-only crawler for simple tasks
+-   **🐳 Docker Deployment**: Easy deployment with FastAPI server and streaming/non-streaming endpoints
+-   **💻 Command-Line Interface**: New `crwl` CLI provides convenient terminal access to all features with intuitive commands and configuration options
+-   **👤 Browser Profiler**: Create and manage persistent browser profiles to save authentication states, cookies, and settings for seamless crawling of protected content
+-   **🧠 Crawl4AI Coding Assistant**: AI-powered coding assistant to answer your question for Crawl4ai, and generate proper code for crawling.
+-   **🏎️ LXML Scraping Mode**: Fast HTML parsing using the `lxml` library for improved performance
+-   **🌐 Proxy Rotation**: Built-in support for proxy switching with `RoundRobinProxyStrategy`
+-   **🤖 LLM Content Filter**: Intelligent markdown generation using LLMs
+-   **📄 PDF Processing**: Extract text, images, and metadata from PDF files
+-   **🔗 URL Redirection Tracking**: Automatically follow and record HTTP redirects
+-   **🤖 LLM Schema Generation**: Easily create extraction schemas with LLM assistance
+-   **🔍 robots.txt Compliance**: Respect website crawling rules
 
-## License 📄
+Read the full details in our [0.5.0 Release Notes](https://docs.crawl4ai.com/blog/releases/0.5.0.html) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
 
-Crawl4AI is released under the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE).
+## Version Numbering in Crawl4AI
 
-## Contact 📧
+Crawl4AI follows standard Python version numbering conventions (PEP 440) to help users understand the stability and features of each release.
+
+### Version Numbers Explained
+
+Our version numbers follow this pattern: `MAJOR.MINOR.PATCH` (e.g., 0.4.3)
+
+#### Pre-release Versions
+We use different suffixes to indicate development stages:
+
+- `dev` (0.4.3dev1): Development versions, unstable
+- `a` (0.4.3a1): Alpha releases, experimental features
+- `b` (0.4.3b1): Beta releases, feature complete but needs testing
+- `rc` (0.4.3rc1): Release candidates, potential final version
+
+#### Installation
+- Regular installation (stable version):
+  ```bash
+  pip install -U crawl4ai
+  ```
+
+- Install pre-release versions:
+  ```bash
+  pip install crawl4ai --pre
+  ```
+
+- Install specific version:
+  ```bash
+  pip install crawl4ai==0.4.3b1
+  ```
+
+#### Why Pre-releases?
+We use pre-releases to:
+- Test new features in real-world scenarios
+- Gather feedback before final releases
+- Ensure stability for production users
+- Allow early adopters to try new features
+
+For production environments, we recommend using the stable version. For testing new features, you can opt-in to pre-releases using the `--pre` flag.
+
+## 📖 Documentation & Roadmap 
+
+> 🚨 **Documentation Update Alert**: We're undertaking a major documentation overhaul next week to reflect recent updates and improvements. Stay tuned for a more comprehensive and up-to-date guide!
+
+For current documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://docs.crawl4ai.com/).
+
+To check our development plans and upcoming features, visit our [Roadmap](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md).
+
+<details>
+<summary>📈 <strong>Development TODOs</strong></summary>
+
+- [x] 0. Graph Crawler: Smart website traversal using graph search algorithms for comprehensive nested page extraction
+- [ ] 1. Question-Based Crawler: Natural language driven web discovery and content extraction
+- [ ] 2. Knowledge-Optimal Crawler: Smart crawling that maximizes knowledge while minimizing data extraction
+- [ ] 3. Agentic Crawler: Autonomous system for complex multi-step crawling operations
+- [ ] 4. Automated Schema Generator: Convert natural language to extraction schemas
+- [ ] 5. Domain-Specific Scrapers: Pre-configured extractors for common platforms (academic, e-commerce)
+- [ ] 6. Web Embedding Index: Semantic search infrastructure for crawled content
+- [ ] 7. Interactive Playground: Web UI for testing, comparing strategies with AI assistance
+- [ ] 8. Performance Monitor: Real-time insights into crawler operations
+- [ ] 9. Cloud Integration: One-click deployment solutions across cloud providers
+- [ ] 10. Sponsorship Program: Structured support system with tiered benefits
+- [ ] 11. Educational Content: "How to Crawl" video series and interactive tutorials
+
+</details>
+
+## 🤝 Contributing 
+
+We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information.
+
+I'll help modify the license section with badges. For the halftone effect, here's a version with it:
+
+Here's the updated license section:
+
+## 📄 License & Attribution
+
+This project is licensed under the Apache License 2.0 with a required attribution clause. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
+
+### Attribution Requirements
+When using Crawl4AI, you must include one of the following attribution methods:
+
+#### 1. Badge Attribution (Recommended)
+Add one of these badges to your README, documentation, or website:
+
+| Theme | Badge |
+|-------|-------|
+| **Disco Theme (Animated)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Night Theme (Dark with Neon)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Dark Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Light Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+ 
+
+HTML code for adding the badges:
+```html
+<!-- Disco Theme (Animated) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Night Theme (Dark with Neon) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Dark Theme (Classic) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Light Theme (Classic) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Simple Shield Badge -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://img.shields.io/badge/Powered%20by-Crawl4AI-blue?style=flat-square" alt="Powered by Crawl4AI"/>
+</a>
+```
+
+#### 2. Text Attribution
+Add this line to your documentation:
+```
+This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
+```
+
+## 📚 Citation
+
+If you use Crawl4AI in your research or project, please cite:
+
+```bibtex
+@software{crawl4ai2024,
+  author = {UncleCode},
+  title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper},
+  year = {2024},
+  publisher = {GitHub},
+  journal = {GitHub Repository},
+  howpublished = {\url{https://github.com/unclecode/crawl4ai}},
+  commit = {Please use the commit hash you're working with}
+}
+```
+
+Text citation format:
+```
+UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software]. 
+GitHub. https://github.com/unclecode/crawl4ai
+```
+
+## 📧 Contact 
 
 For questions, suggestions, or feedback, feel free to reach out:
 
@@ -165,6 +685,31 @@ For questions, suggestions, or feedback, feel free to reach out:
 
 Happy Crawling! 🕸️🚀
 
+## 🗾 Mission
+
+Our mission is to unlock the value of personal and enterprise data by transforming digital footprints into structured, tradeable assets. Crawl4AI empowers individuals and organizations with open-source tools to extract and structure data, fostering a shared data economy.  
+
+We envision a future where AI is powered by real human knowledge, ensuring data creators directly benefit from their contributions. By democratizing data and enabling ethical sharing, we are laying the foundation for authentic AI advancement.
+
+<details>
+<summary>🔑 <strong>Key Opportunities</strong></summary>
+ 
+- **Data Capitalization**: Transform digital footprints into measurable, valuable assets.  
+- **Authentic AI Data**: Provide AI systems with real human insights.  
+- **Shared Economy**: Create a fair data marketplace that benefits data creators.  
+
+</details>
+
+<details>
+<summary>🚀 <strong>Development Pathway</strong></summary>
+
+1. **Open-Source Tools**: Community-driven platforms for transparent data extraction.  
+2. **Digital Asset Structuring**: Tools to organize and value digital knowledge.  
+3. **Ethical Data Marketplace**: A secure, fair platform for exchanging structured data.  
+
+For more details, see our [full mission statement](./MISSION.md).
+</details>
+
 ## Star History
 
-[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)
+[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

ROADMAP.md

@@ -0,0 +1,503 @@
+# Crawl4AI Strategic Roadmap
+
+```mermaid
+%%{init: {'themeVariables': { 'fontSize': '14px'}}}%%
+graph TD
+    subgraph A1[Advanced Crawling Systems 🔧]
+        A["`
+        • Graph Crawler ✓
+        • Question-Based Crawler
+        • Knowledge-Optimal Crawler
+        • Agentic Crawler
+        `"]
+    end
+
+    subgraph A2[Specialized Features 🛠️]
+        B["`
+        • Automated Schema Generator
+        • Domain-Specific Scrapers
+        • 
+        • 
+        `"]
+    end
+
+    subgraph A3[Development Tools 🔨]
+        C["`
+        • Interactive Playground
+        • Performance Monitor
+        • Cloud Integration
+        • 
+        `"]
+    end
+
+    subgraph A4[Community & Growth 🌱]
+        D["`
+        • Sponsorship Program
+        • Educational Content
+        • 
+        • 
+        `"]
+    end
+
+    classDef default fill:#f9f9f9,stroke:#333,stroke-width:2px
+    classDef section fill:#f0f0f0,stroke:#333,stroke-width:4px,rx:10
+    class A1,A2,A3,A4 section
+
+    %% Layout hints
+    A1 --> A2[" "]
+    A3 --> A4[" "]
+    linkStyle 0,1 stroke:none
+```
+
+Crawl4AI is evolving to provide more intelligent, efficient, and versatile web crawling capabilities. This roadmap outlines the key developments and features planned for the project, organized into strategic sections that build upon our current foundation.
+
+## 1. Advanced Crawling Systems 🔧
+
+This section introduces three powerful crawling systems that extend Crawl4AI's capabilities from basic web crawling to intelligent, purpose-driven data extraction.
+
+### 1.1 Question-Based Crawler
+The Question-Based Crawler enhances our core engine by enabling automatic discovery and extraction of relevant web content based on natural language questions.
+
+Key Features:
+- SerpiAPI integration for intelligent web search
+- Relevancy scoring for search results
+- Automatic URL discovery and prioritization
+- Cross-source validation
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.discovery import QuestionBasedDiscovery
+
+async with AsyncWebCrawler() as crawler:
+    discovery = QuestionBasedDiscovery(crawler)
+    results = await discovery.arun(
+        question="What are the system requirements for major cloud providers' GPU instances?",
+        max_urls=5,
+        relevance_threshold=0.7
+    )
+    
+    for result in results:
+        print(f"Source: {result.url} (Relevance: {result.relevance_score})")
+        print(f"Content: {result.markdown}\n")
+```
+
+### 1.2 Knowledge-Optimal Crawler
+An intelligent crawling system that solves the optimization problem of minimizing data extraction while maximizing knowledge acquisition for specific objectives.
+
+Key Features:
+- Smart content prioritization
+- Minimal data extraction for maximum knowledge
+- Probabilistic relevance assessment
+- Objective-driven crawling paths
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.optimization import KnowledgeOptimizer
+
+async with AsyncWebCrawler() as crawler:
+    optimizer = KnowledgeOptimizer(
+        objective="Understand GPU instance pricing and limitations across cloud providers",
+        required_knowledge=[
+            "pricing structure",
+            "GPU specifications",
+            "usage limits",
+            "availability zones"
+        ],
+        confidence_threshold=0.85
+    )
+    
+    result = await crawler.arun(
+        urls=[
+            "https://aws.amazon.com/ec2/pricing/",
+            "https://cloud.google.com/gpu",
+            "https://azure.microsoft.com/pricing/"
+        ],
+        optimizer=optimizer,
+        optimization_mode="minimal_extraction"
+    )
+    
+    print(f"Knowledge Coverage: {result.knowledge_coverage}")
+    print(f"Data Efficiency: {result.efficiency_ratio}")
+    print(f"Extracted Content: {result.optimal_content}")
+```
+
+### 1.3 Agentic Crawler
+An autonomous system capable of understanding complex goals and automatically planning and executing multi-step crawling operations.
+
+Key Features:
+- Autonomous goal interpretation
+- Dynamic step planning
+- Interactive navigation capabilities
+- Visual recognition and interaction
+- Automatic error recovery
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.agents import CrawlerAgent
+
+async with AsyncWebCrawler() as crawler:
+    agent = CrawlerAgent(crawler)
+    
+    # Automatic planning and execution
+    result = await agent.arun(
+        goal="Find research papers about quantum computing published in 2023 with more than 50 citations",
+        auto_retry=True
+    )
+    print("Generated Plan:", result.executed_steps)
+    print("Extracted Data:", result.data)
+    
+    # Using custom steps with automatic execution
+    result = await agent.arun(
+        goal="Extract conference deadlines from ML conferences",
+        custom_plan=[
+            "Navigate to conference page",
+            "Find important dates section",
+            "Extract submission deadlines",
+            "Verify dates are for 2024"
+        ]
+    )
+    
+    # Monitoring execution
+    print("Step Completion:", result.step_status)
+    print("Execution Time:", result.execution_time)
+    print("Success Rate:", result.success_rate)
+```
+
+# Section 2: Specialized Features 🛠️
+
+This section introduces specialized tools and features that enhance Crawl4AI's capabilities for specific use cases and data extraction needs.
+
+### 2.1 Automated Schema Generator
+A system that automatically generates JsonCssExtractionStrategy schemas from natural language descriptions, making structured data extraction accessible to all users.
+
+Key Features:
+- Natural language schema generation
+- Automatic pattern detection
+- Predefined schema templates
+- Chrome extension for visual schema building
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.schema import SchemaGenerator
+
+# Generate schema from natural language description
+generator = SchemaGenerator()
+schema = await generator.generate(
+    url="https://news-website.com",
+    description="For each news article on the page, I need the headline, publication date, and main image"
+)
+
+# Use generated schema with crawler
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url="https://news-website.com",
+        extraction_strategy=schema
+    )
+
+# Example of generated schema:
+"""
+{
+    "name": "News Article Extractor",
+    "baseSelector": "article.news-item",
+    "fields": [
+        {
+            "name": "headline",
+            "selector": "h2.article-title",
+            "type": "text"
+        },
+        {
+            "name": "date",
+            "selector": "span.publish-date",
+            "type": "text"
+        },
+        {
+            "name": "image",
+            "selector": "img.article-image",
+            "type": "attribute",
+            "attribute": "src"
+        }
+    ]
+}
+"""
+```
+
+### 2.2 Domain Specific Scrapers
+Specialized extraction strategies optimized for common website types and platforms, providing consistent and reliable data extraction without additional configuration.
+
+Key Features:
+- Pre-configured extractors for popular platforms
+- Academic site specialization (arXiv, NCBI)
+- E-commerce standardization
+- Documentation site handling
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.extractors import AcademicExtractor, EcommerceExtractor
+
+async with AsyncWebCrawler() as crawler:
+    # Academic paper extraction
+    papers = await crawler.arun(
+        url="https://arxiv.org/list/cs.AI/recent",
+        extractor="academic",  # Built-in extractor type
+        site_type="arxiv",     # Specific site optimization
+        extract_fields=[
+            "title", 
+            "authors", 
+            "abstract", 
+            "citations"
+        ]
+    )
+    
+    # E-commerce product data
+    products = await crawler.arun(
+        url="https://store.example.com/products",
+        extractor="ecommerce",
+        extract_fields=[
+            "name",
+            "price",
+            "availability",
+            "reviews"
+        ]
+    )
+```
+
+### 2.3 Web Embedding Index
+Creates and maintains a semantic search infrastructure for crawled content, enabling efficient retrieval and querying of web content through vector embeddings.
+
+Key Features:
+- Automatic embedding generation
+- Intelligent content chunking
+- Efficient vector storage and indexing
+- Semantic search capabilities
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.indexing import WebIndex
+
+# Initialize and build index
+index = WebIndex(model="efficient-mini")
+
+async with AsyncWebCrawler() as crawler:
+    # Crawl and index content
+    await index.build(
+        urls=["https://docs.example.com"],
+        crawler=crawler,
+        options={
+            "chunk_method": "semantic",
+            "update_policy": "incremental",
+            "embedding_batch_size": 100
+        }
+    )
+
+    # Search through indexed content
+    results = await index.search(
+        query="How to implement OAuth authentication?",
+        filters={
+            "content_type": "technical",
+            "recency": "6months"
+        },
+        top_k=5
+    )
+
+    # Get similar content
+    similar = await index.find_similar(
+        url="https://docs.example.com/auth/oauth",
+        threshold=0.85
+    )
+```
+
+Each of these specialized features builds upon Crawl4AI's core functionality while providing targeted solutions for specific use cases. They can be used independently or combined for more complex data extraction and processing needs.
+
+# Section 3: Development Tools 🔧
+
+This section covers tools designed to enhance the development experience, monitoring, and deployment of Crawl4AI applications.
+
+### 3.1 Crawl4AI Playground 🎮
+
+The Crawl4AI Playground is an interactive web-based development environment that simplifies web scraping experimentation, development, and deployment. With its intuitive interface and AI-powered assistance, users can quickly prototype, test, and deploy web scraping solutions.
+
+#### Key Features 🌟
+
+##### Visual Strategy Builder
+- Interactive point-and-click interface for building extraction strategies
+- Real-time preview of selected elements
+- Side-by-side comparison of different extraction approaches
+- Visual validation of CSS selectors and XPath queries
+
+##### AI Assistant Integration
+- Strategy recommendations based on target website analysis
+- Parameter optimization suggestions
+- Best practices guidance for specific use cases
+- Automated error detection and resolution
+- Performance optimization tips
+
+##### Real-Time Testing & Validation
+- Live preview of extraction results
+- Side-by-side comparison of multiple strategies
+- Performance metrics visualization
+- Automatic validation of extracted data
+- Error detection and debugging tools
+
+##### Project Management
+- Save and organize multiple scraping projects
+- Version control for configurations
+- Export/import project settings
+- Share configurations with team members
+- Project templates for common use cases
+
+##### Deployment Pipeline
+- One-click deployment to various environments
+- Docker container generation
+- Cloud deployment templates (AWS, GCP, Azure)
+- Scaling configuration management
+- Monitoring setup automation
+
+
+### 3.2 Performance Monitoring System
+A comprehensive monitoring solution providing real-time insights into crawler operations, resource usage, and system health through both CLI and GUI interfaces.
+
+Key Features:
+- Real-time resource tracking
+- Active crawl monitoring
+- Performance statistics
+- Customizable alerting system
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.monitor import CrawlMonitor
+
+# Initialize monitoring
+monitor = CrawlMonitor()
+
+# Start monitoring with CLI interface
+await monitor.start(
+    mode="cli",  # or "gui"
+    refresh_rate="1s",
+    metrics={
+        "resources": ["cpu", "memory", "network"],
+        "crawls": ["active", "queued", "completed"],
+        "performance": ["success_rate", "response_times"]
+    }
+)
+
+# Example CLI output:
+"""
+Crawl4AI Monitor (Live) - Press Q to exit
+────────────────────────────────────────
+System Usage:
+ ├─ CPU: ███████░░░ 70%
+ └─ Memory: ████░░░░░ 2.1GB/8GB
+
+Active Crawls:
+ID    URL                   Status    Progress
+001   docs.example.com     🟢 Active   75%
+002   api.service.com      🟡 Queue    -
+
+Metrics (Last 5min):
+ ├─ Success Rate: 98%
+ ├─ Avg Response: 0.6s
+ └─ Pages/sec: 8.5
+"""
+```
+
+### 3.3 Cloud Integration
+Streamlined deployment tools for setting up Crawl4AI in various cloud environments, with support for scaling and monitoring.
+
+Key Features:
+- One-click deployment solutions
+- Auto-scaling configuration
+- Load balancing setup
+- Cloud-specific optimizations
+- Monitoring integration
+
+```python
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.deploy import CloudDeployer
+
+# Initialize deployer
+deployer = CloudDeployer()
+
+# Deploy crawler service
+deployment = await deployer.deploy(
+    service_name="crawler-cluster",
+    platform="aws",  # or "gcp", "azure"
+    config={
+        "instance_type": "compute-optimized",
+        "auto_scaling": {
+            "min_instances": 2,
+            "max_instances": 10,
+            "scale_based_on": "cpu_usage"
+        },
+        "region": "us-east-1",
+        "monitoring": True
+    }
+)
+
+# Get deployment status and endpoints
+print(f"Service Status: {deployment.status}")
+print(f"API Endpoint: {deployment.endpoint}")
+print(f"Monitor URL: {deployment.monitor_url}")
+```
+
+These development tools work together to provide a comprehensive environment for developing, testing, monitoring, and deploying Crawl4AI applications. The Playground helps users experiment and generate optimal configurations, the Performance Monitor ensures smooth operation, and the Cloud Integration tools simplify deployment and scaling.
+
+# Section 4: Community & Growth 🌱
+
+This section outlines initiatives designed to build and support the Crawl4AI community, provide educational resources, and ensure sustainable project growth.
+
+### 4.1 Sponsorship Program
+A structured program to support ongoing development and maintenance of Crawl4AI while providing valuable benefits to sponsors.
+
+Key Features:
+- Multiple sponsorship tiers
+- Sponsor recognition system
+- Priority support for sponsors
+- Early access to new features
+- Custom feature development opportunities
+
+Program Structure (not yet finalized):
+```
+Sponsorship Tiers:
+
+🥉 Bronze Supporter
+- GitHub Sponsor badge
+- Priority issue response
+- Community Discord role
+
+🥈 Silver Supporter
+- All Bronze benefits
+- Technical support channel
+- Vote on roadmap priorities
+- Early access to beta features
+
+🥇 Gold Supporter
+- All Silver benefits
+- Custom feature requests
+- Direct developer access
+- Private support sessions
+
+💎 Diamond Partner
+- All Gold benefits
+- Custom development
+- On-demand consulting
+- Integration support
+```
+
+### 4.2 "How to Crawl" Video Series
+A comprehensive educational resource teaching users how to effectively use Crawl4AI for various web scraping and data extraction scenarios.
+
+Key Features:
+- Step-by-step tutorials
+- Real-world use cases
+- Best practices
+- Integration guides
+- Advanced feature deep-dives
+
+These community initiatives are designed to:
+- Provide comprehensive learning resources
+- Foster a supportive user community
+- Ensure sustainable project development
+- Share knowledge and best practices
+- Create opportunities for collaboration
+
+The combination of structured support through sponsorship, educational content through video series, and interactive learning through the playground creates a robust ecosystem for both new and experienced users of Crawl4AI.

cliff.toml

@@ -0,0 +1,24 @@
+[changelog]
+# Template format
+header = """
+# Changelog\n
+All notable changes to this project will be documented in this file.\n
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n
+"""
+
+# Organize commits by type
+[git]
+conventional_commits = true
+filter_unconventional = true
+commit_parsers = [
+    { message = "^feat", group = "Added"},
+    { message = "^fix", group = "Fixed"},
+    { message = "^doc", group = "Documentation"},
+    { message = "^perf", group = "Performance"},
+    { message = "^refactor", group = "Changed"},
+    { message = "^style", group = "Changed"},
+    { message = "^test", group = "Testing"},
+    { message = "^chore\\(release\\): prepare for", skip = true},
+    { message = "^chore", group = "Miscellaneous Tasks"},
+]

crawl4ai/__init__.py

@@ -1 +1,165 @@
-from .web_crawler import WebCrawler
+# __init__.py
+import warnings
+
+from .async_webcrawler import AsyncWebCrawler, CacheMode
+from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig, LLMConfig
+
+from .pipeline.pipeline import (
+    Pipeline,
+    create_pipeline,
+)
+from .pipeline.crawler import Crawler
+
+from .content_scraping_strategy import (
+    ContentScrapingStrategy,
+    WebScrapingStrategy,
+    LXMLWebScrapingStrategy,
+)
+from .async_logger import (
+    AsyncLoggerBase,
+    AsyncLogger,
+)
+from .proxy_strategy import (
+    ProxyRotationStrategy,
+    RoundRobinProxyStrategy,
+)
+from .extraction_strategy import (
+    ExtractionStrategy,
+    LLMExtractionStrategy,
+    CosineStrategy,
+    JsonCssExtractionStrategy,
+    JsonXPathExtractionStrategy,
+    JsonLxmlExtractionStrategy
+)
+from .chunking_strategy import ChunkingStrategy, RegexChunking
+from .markdown_generation_strategy import DefaultMarkdownGenerator
+from .content_filter_strategy import (
+    PruningContentFilter,
+    BM25ContentFilter,
+    LLMContentFilter,
+    RelevantContentFilter,
+)
+from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
+from .components.crawler_monitor import CrawlerMonitor
+from .async_dispatcher import (
+    MemoryAdaptiveDispatcher,
+    SemaphoreDispatcher,
+    RateLimiter,
+    BaseDispatcher,
+)
+from .docker_client import Crawl4aiDockerClient
+from .hub import CrawlerHub
+from .browser_profiler import BrowserProfiler
+from .deep_crawling import (
+    DeepCrawlStrategy,
+    BFSDeepCrawlStrategy,
+    FilterChain,
+    URLPatternFilter,
+    DomainFilter,
+    ContentTypeFilter,
+    URLFilter,
+    FilterStats,
+    SEOFilter,
+    KeywordRelevanceScorer,
+    URLScorer,
+    CompositeScorer,
+    DomainAuthorityScorer,
+    FreshnessScorer,
+    PathDepthScorer,
+    BestFirstCrawlingStrategy,
+    DFSDeepCrawlStrategy,
+    DeepCrawlDecorator,
+)
+
+from .async_crawler_strategy import AsyncPlaywrightCrawlerStrategy, AsyncHTTPCrawlerStrategy
+
+__all__ = [
+    "Pipeline",
+    "AsyncPlaywrightCrawlerStrategy",
+    "AsyncHTTPCrawlerStrategy",
+    "create_pipeline",
+    "Crawler",
+    "AsyncLoggerBase",
+    "AsyncLogger",
+    "AsyncWebCrawler",
+    "BrowserProfiler",
+    "LLMConfig",
+    "DeepCrawlStrategy",
+    "BFSDeepCrawlStrategy",
+    "BestFirstCrawlingStrategy",
+    "DFSDeepCrawlStrategy",
+    "FilterChain",
+    "URLPatternFilter",
+    "ContentTypeFilter",
+    "DomainFilter",
+    "FilterStats",
+    "URLFilter",
+    "SEOFilter",
+    "KeywordRelevanceScorer",
+    "URLScorer",
+    "CompositeScorer",
+    "DomainAuthorityScorer",
+    "FreshnessScorer",
+    "PathDepthScorer",
+    "DeepCrawlDecorator",
+    "CrawlResult",
+    "CrawlerHub",
+    "CacheMode",
+    "ContentScrapingStrategy",
+    "WebScrapingStrategy",
+    "LXMLWebScrapingStrategy",
+    "BrowserConfig",
+    "CrawlerRunConfig",
+    "HTTPCrawlerConfig",
+    "ExtractionStrategy",
+    "LLMExtractionStrategy",
+    "CosineStrategy",
+    "JsonCssExtractionStrategy",
+    "JsonXPathExtractionStrategy",
+    "JsonLxmlExtractionStrategy",
+    "ChunkingStrategy",
+    "RegexChunking",
+    "DefaultMarkdownGenerator",
+    "RelevantContentFilter",
+    "PruningContentFilter",
+    "BM25ContentFilter",
+    "LLMContentFilter",
+    "BaseDispatcher",
+    "MemoryAdaptiveDispatcher",
+    "SemaphoreDispatcher",
+    "RateLimiter",
+    "CrawlerMonitor",
+    "DisplayMode",
+    "MarkdownGenerationResult",
+    "Crawl4aiDockerClient",
+    "ProxyRotationStrategy",
+    "RoundRobinProxyStrategy",
+]
+
+
+# def is_sync_version_installed():
+#     try:
+#         import selenium # noqa
+
+#         return True
+#     except ImportError:
+#         return False
+
+
+# if is_sync_version_installed():
+#     try:
+#         from .web_crawler import WebCrawler
+
+#         __all__.append("WebCrawler")
+#     except ImportError:
+#         print(
+#             "Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies."
+#         )
+# else:
+#     WebCrawler = None
+#     # import warnings
+#     # print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.")
+
+# Disable all Pydantic warnings
+warnings.filterwarnings("ignore", module="pydantic")
+# pydantic_warnings.filter_warnings()

crawl4ai/__version__.py

@@ -0,0 +1,2 @@
+# crawl4ai/_version.py
+__version__ = "0.5.0.post8"

crawl4ai/async_database.py

@@ -0,0 +1,558 @@
+import os
+from pathlib import Path
+import aiosqlite
+import asyncio
+from typing import Optional, Dict
+from contextlib import asynccontextmanager
+import json  
+from .models import CrawlResult, MarkdownGenerationResult, StringCompatibleMarkdown
+import aiofiles
+from .async_logger import AsyncLogger
+
+from .utils import ensure_content_dirs, generate_content_hash
+from .utils import VersionManager
+from .utils import get_error_context, create_box_message
+
+base_directory = DB_PATH = os.path.join(
+    os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
+)
+os.makedirs(DB_PATH, exist_ok=True)
+DB_PATH = os.path.join(base_directory, "crawl4ai.db")
+
+
+class AsyncDatabaseManager:
+    def __init__(self, pool_size: int = 10, max_retries: int = 3):
+        self.db_path = DB_PATH
+        self.content_paths = ensure_content_dirs(os.path.dirname(DB_PATH))
+        self.pool_size = pool_size
+        self.max_retries = max_retries
+        self.connection_pool: Dict[int, aiosqlite.Connection] = {}
+        self.pool_lock = asyncio.Lock()
+        self.init_lock = asyncio.Lock()
+        self.connection_semaphore = asyncio.Semaphore(pool_size)
+        self._initialized = False
+        self.version_manager = VersionManager()
+        self.logger = AsyncLogger(
+            log_file=os.path.join(base_directory, ".crawl4ai", "crawler_db.log"),
+            verbose=False,
+            tag_width=10,
+        )
+
+    async def initialize(self):
+        """Initialize the database and connection pool"""
+        try:
+            self.logger.info("Initializing database", tag="INIT")
+            # Ensure the database file exists
+            os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
+
+            # Check if version update is needed
+            needs_update = self.version_manager.needs_update()
+
+            # Always ensure base table exists
+            await self.ainit_db()
+
+            # Verify the table exists
+            async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
+                async with db.execute(
+                    "SELECT name FROM sqlite_master WHERE type='table' AND name='crawled_data'"
+                ) as cursor:
+                    result = await cursor.fetchone()
+                    if not result:
+                        raise Exception("crawled_data table was not created")
+
+            # If version changed or fresh install, run updates
+            if needs_update:
+                self.logger.info("New version detected, running updates", tag="INIT")
+                await self.update_db_schema()
+                from .migrations import (
+                    run_migration,
+                )  # Import here to avoid circular imports
+
+                await run_migration()
+                self.version_manager.update_version()  # Update stored version after successful migration
+                self.logger.success(
+                    "Version update completed successfully", tag="COMPLETE"
+                )
+            else:
+                self.logger.success(
+                    "Database initialization completed successfully", tag="COMPLETE"
+                )
+
+        except Exception as e:
+            self.logger.error(
+                message="Database initialization error: {error}",
+                tag="ERROR",
+                params={"error": str(e)},
+            )
+            self.logger.info(
+                message="Database will be initialized on first use", tag="INIT"
+            )
+
+            raise
+
+    async def cleanup(self):
+        """Cleanup connections when shutting down"""
+        async with self.pool_lock:
+            for conn in self.connection_pool.values():
+                await conn.close()
+            self.connection_pool.clear()
+
+    @asynccontextmanager
+    async def get_connection(self):
+        """Connection pool manager with enhanced error handling"""
+        if not self._initialized:
+            async with self.init_lock:
+                if not self._initialized:
+                    try:
+                        await self.initialize()
+                        self._initialized = True
+                    except Exception as e:
+                        import sys
+
+                        error_context = get_error_context(sys.exc_info())
+                        self.logger.error(
+                            message="Database initialization failed:\n{error}\n\nContext:\n{context}\n\nTraceback:\n{traceback}",
+                            tag="ERROR",
+                            force_verbose=True,
+                            params={
+                                "error": str(e),
+                                "context": error_context["code_context"],
+                                "traceback": error_context["full_traceback"],
+                            },
+                        )
+                        raise
+
+        await self.connection_semaphore.acquire()
+        task_id = id(asyncio.current_task())
+
+        try:
+            async with self.pool_lock:
+                if task_id not in self.connection_pool:
+                    try:
+                        conn = await aiosqlite.connect(self.db_path, timeout=30.0)
+                        await conn.execute("PRAGMA journal_mode = WAL")
+                        await conn.execute("PRAGMA busy_timeout = 5000")
+
+                        # Verify database structure
+                        async with conn.execute(
+                            "PRAGMA table_info(crawled_data)"
+                        ) as cursor:
+                            columns = await cursor.fetchall()
+                            column_names = [col[1] for col in columns]
+                            expected_columns = {
+                                "url",
+                                "html",
+                                "cleaned_html",
+                                "markdown",
+                                "extracted_content",
+                                "success",
+                                "media",
+                                "links",
+                                "metadata",
+                                "screenshot",
+                                "response_headers",
+                                "downloaded_files",
+                            }
+                            missing_columns = expected_columns - set(column_names)
+                            if missing_columns:
+                                raise ValueError(
+                                    f"Database missing columns: {missing_columns}"
+                                )
+
+                        self.connection_pool[task_id] = conn
+                    except Exception as e:
+                        import sys
+
+                        error_context = get_error_context(sys.exc_info())
+                        error_message = (
+                            f"Unexpected error in db get_connection at line {error_context['line_no']} "
+                            f"in {error_context['function']} ({error_context['filename']}):\n"
+                            f"Error: {str(e)}\n\n"
+                            f"Code context:\n{error_context['code_context']}"
+                        )
+                        self.logger.error(
+                            message=create_box_message(error_message, type="error"),
+                        )
+
+                        raise
+
+            yield self.connection_pool[task_id]
+
+        except Exception as e:
+            import sys
+
+            error_context = get_error_context(sys.exc_info())
+            error_message = (
+                f"Unexpected error in db get_connection at line {error_context['line_no']} "
+                f"in {error_context['function']} ({error_context['filename']}):\n"
+                f"Error: {str(e)}\n\n"
+                f"Code context:\n{error_context['code_context']}"
+            )
+            self.logger.error(
+                message=create_box_message(error_message, type="error"),
+            )
+            raise
+        finally:
+            async with self.pool_lock:
+                if task_id in self.connection_pool:
+                    await self.connection_pool[task_id].close()
+                    del self.connection_pool[task_id]
+            self.connection_semaphore.release()
+
+    async def execute_with_retry(self, operation, *args):
+        """Execute database operations with retry logic"""
+        for attempt in range(self.max_retries):
+            try:
+                async with self.get_connection() as db:
+                    result = await operation(db, *args)
+                    await db.commit()
+                    return result
+            except Exception as e:
+                if attempt == self.max_retries - 1:
+                    self.logger.error(
+                        message="Operation failed after {retries} attempts: {error}",
+                        tag="ERROR",
+                        force_verbose=True,
+                        params={"retries": self.max_retries, "error": str(e)},
+                    )
+                    raise
+                await asyncio.sleep(1 * (attempt + 1))  # Exponential backoff
+
+    async def ainit_db(self):
+        """Initialize database schema"""
+        async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
+            await db.execute(
+                """
+                CREATE TABLE IF NOT EXISTS crawled_data (
+                    url TEXT PRIMARY KEY,
+                    html TEXT,
+                    cleaned_html TEXT,
+                    markdown TEXT,
+                    extracted_content TEXT,
+                    success BOOLEAN,
+                    media TEXT DEFAULT "{}",
+                    links TEXT DEFAULT "{}",
+                    metadata TEXT DEFAULT "{}",
+                    screenshot TEXT DEFAULT "",
+                    response_headers TEXT DEFAULT "{}",
+                    downloaded_files TEXT DEFAULT "{}"  -- New column added
+                )
+            """
+            )
+            await db.commit()
+
+    async def update_db_schema(self):
+        """Update database schema if needed"""
+        async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
+            cursor = await db.execute("PRAGMA table_info(crawled_data)")
+            columns = await cursor.fetchall()
+            column_names = [column[1] for column in columns]
+
+            # List of new columns to add
+            new_columns = [
+                "media",
+                "links",
+                "metadata",
+                "screenshot",
+                "response_headers",
+                "downloaded_files",
+            ]
+
+            for column in new_columns:
+                if column not in column_names:
+                    await self.aalter_db_add_column(column, db)
+            await db.commit()
+
+    async def aalter_db_add_column(self, new_column: str, db):
+        """Add new column to the database"""
+        if new_column == "response_headers":
+            await db.execute(
+                f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT "{{}}"'
+            )
+        else:
+            await db.execute(
+                f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""'
+            )
+        self.logger.info(
+            message="Added column '{column}' to the database",
+            tag="INIT",
+            params={"column": new_column},
+        )
+
+    async def aget_cached_url(self, url: str) -> Optional[CrawlResult]:
+        """Retrieve cached URL data as CrawlResult"""
+
+        async def _get(db):
+            async with db.execute(
+                "SELECT * FROM crawled_data WHERE url = ?", (url,)
+            ) as cursor:
+                row = await cursor.fetchone()
+                if not row:
+                    return None
+
+                # Get column names
+                columns = [description[0] for description in cursor.description]
+                # Create dict from row data
+                row_dict = dict(zip(columns, row))
+
+                # Load content from files using stored hashes
+                content_fields = {
+                    "html": row_dict["html"],
+                    "cleaned_html": row_dict["cleaned_html"],
+                    "markdown": row_dict["markdown"],
+                    "extracted_content": row_dict["extracted_content"],
+                    "screenshot": row_dict["screenshot"],
+                    "screenshots": row_dict["screenshot"],
+                }
+
+                for field, hash_value in content_fields.items():
+                    if hash_value:
+                        content = await self._load_content(
+                            hash_value,
+                            field.split("_")[0],  # Get content type from field name
+                        )
+                        row_dict[field] = content or ""
+                    else:
+                        row_dict[field] = ""
+
+                # Parse JSON fields
+                json_fields = [
+                    "media",
+                    "links",
+                    "metadata",
+                    "response_headers",
+                    "markdown",
+                ]
+                for field in json_fields:
+                    try:
+                        row_dict[field] = (
+                            json.loads(row_dict[field]) if row_dict[field] else {}
+                        )
+                    except json.JSONDecodeError:
+                        # Very UGLY, never mention it to me please
+                        if field == "markdown" and isinstance(row_dict[field], str):
+                            row_dict[field] = MarkdownGenerationResult(
+                                raw_markdown=row_dict[field] or "",
+                                markdown_with_citations="",
+                                references_markdown="",
+                                fit_markdown="",
+                                fit_html="",
+                            )
+                        else:
+                            row_dict[field] = {}
+
+                if isinstance(row_dict["markdown"], Dict):
+                    if row_dict["markdown"].get("raw_markdown"):
+                        row_dict["markdown"] = row_dict["markdown"]["raw_markdown"]
+
+                # Parse downloaded_files
+                try:
+                    row_dict["downloaded_files"] = (
+                        json.loads(row_dict["downloaded_files"])
+                        if row_dict["downloaded_files"]
+                        else []
+                    )
+                except json.JSONDecodeError:
+                    row_dict["downloaded_files"] = []
+
+                # Remove any fields not in CrawlResult model
+                valid_fields = CrawlResult.__annotations__.keys()
+                filtered_dict = {k: v for k, v in row_dict.items() if k in valid_fields}
+                filtered_dict["markdown"] = row_dict["markdown"]
+                return CrawlResult(**filtered_dict)
+
+        try:
+            return await self.execute_with_retry(_get)
+        except Exception as e:
+            self.logger.error(
+                message="Error retrieving cached URL: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)},
+            )
+            return None
+
+    async def acache_url(self, result: CrawlResult):
+        """Cache CrawlResult data"""
+        # Store content files and get hashes
+        content_map = {
+            "html": (result.html, "html"),
+            "cleaned_html": (result.cleaned_html or "", "cleaned"),
+            "markdown": None,
+            "extracted_content": (result.extracted_content or "", "extracted"),
+            "screenshot": (result.screenshot or "", "screenshots"),
+        }
+
+        try:
+            if isinstance(result.markdown, StringCompatibleMarkdown):
+                content_map["markdown"] = (
+                    result.markdown,
+                    "markdown",
+                )
+            elif isinstance(result.markdown, MarkdownGenerationResult):
+                content_map["markdown"] = (
+                    result.markdown.model_dump_json(),
+                    "markdown",
+                )
+            elif isinstance(result.markdown, str):
+                markdown_result = MarkdownGenerationResult(raw_markdown=result.markdown)
+                content_map["markdown"] = (
+                    markdown_result.model_dump_json(),
+                    "markdown",
+                )
+            else:
+                content_map["markdown"] = (
+                    MarkdownGenerationResult().model_dump_json(),
+                    "markdown",
+                )
+        except Exception as e:
+            self.logger.warning(
+                message=f"Error processing markdown content: {str(e)}", tag="WARNING"
+            )
+            # Fallback to empty markdown result
+            content_map["markdown"] = (
+                MarkdownGenerationResult().model_dump_json(),
+                "markdown",
+            )
+
+        content_hashes = {}
+        for field, (content, content_type) in content_map.items():
+            content_hashes[field] = await self._store_content(content, content_type)
+
+        async def _cache(db):
+            await db.execute(
+                """
+                INSERT INTO crawled_data (
+                    url, html, cleaned_html, markdown,
+                    extracted_content, success, media, links, metadata,
+                    screenshot, response_headers, downloaded_files
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(url) DO UPDATE SET
+                    html = excluded.html,
+                    cleaned_html = excluded.cleaned_html,
+                    markdown = excluded.markdown,
+                    extracted_content = excluded.extracted_content,
+                    success = excluded.success,
+                    media = excluded.media,
+                    links = excluded.links,
+                    metadata = excluded.metadata,
+                    screenshot = excluded.screenshot,
+                    response_headers = excluded.response_headers,
+                    downloaded_files = excluded.downloaded_files
+            """,
+                (
+                    result.url,
+                    content_hashes["html"],
+                    content_hashes["cleaned_html"],
+                    content_hashes["markdown"],
+                    content_hashes["extracted_content"],
+                    result.success,
+                    json.dumps(result.media),
+                    json.dumps(result.links),
+                    json.dumps(result.metadata or {}),
+                    content_hashes["screenshot"],
+                    json.dumps(result.response_headers or {}),
+                    json.dumps(result.downloaded_files or []),
+                ),
+            )
+
+        try:
+            await self.execute_with_retry(_cache)
+        except Exception as e:
+            self.logger.error(
+                message="Error caching URL: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)},
+            )
+
+    async def aget_total_count(self) -> int:
+        """Get total number of cached URLs"""
+
+        async def _count(db):
+            async with db.execute("SELECT COUNT(*) FROM crawled_data") as cursor:
+                result = await cursor.fetchone()
+                return result[0] if result else 0
+
+        try:
+            return await self.execute_with_retry(_count)
+        except Exception as e:
+            self.logger.error(
+                message="Error getting total count: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)},
+            )
+            return 0
+
+    async def aclear_db(self):
+        """Clear all data from the database"""
+
+        async def _clear(db):
+            await db.execute("DELETE FROM crawled_data")
+
+        try:
+            await self.execute_with_retry(_clear)
+        except Exception as e:
+            self.logger.error(
+                message="Error clearing database: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)},
+            )
+
+    async def aflush_db(self):
+        """Drop the entire table"""
+
+        async def _flush(db):
+            await db.execute("DROP TABLE IF EXISTS crawled_data")
+
+        try:
+            await self.execute_with_retry(_flush)
+        except Exception as e:
+            self.logger.error(
+                message="Error flushing database: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)},
+            )
+
+    async def _store_content(self, content: str, content_type: str) -> str:
+        """Store content in filesystem and return hash"""
+        if not content:
+            return ""
+
+        content_hash = generate_content_hash(content)
+        file_path = os.path.join(self.content_paths[content_type], content_hash)
+
+        # Only write if file doesn't exist
+        if not os.path.exists(file_path):
+            async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
+                await f.write(content)
+
+        return content_hash
+
+    async def _load_content(
+        self, content_hash: str, content_type: str
+    ) -> Optional[str]:
+        """Load content from filesystem by hash"""
+        if not content_hash:
+            return None
+
+        file_path = os.path.join(self.content_paths[content_type], content_hash)
+        try:
+            async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
+                return await f.read()
+        except:
+            self.logger.error(
+                message="Failed to load content: {file_path}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"file_path": file_path},
+            )
+            return None
+
+
+# Create a singleton instance
+async_db_manager = AsyncDatabaseManager()

crawl4ai/async_dispatcher.py

@@ -0,0 +1,646 @@
+from typing import Dict, Optional, List, Tuple
+from .async_configs import CrawlerRunConfig
+from .models import (
+    CrawlResult,
+    CrawlerTaskResult,
+    CrawlStatus,
+    DomainState,
+)
+
+from .components.crawler_monitor import CrawlerMonitor
+
+from .types import AsyncWebCrawler
+
+from collections.abc import AsyncGenerator
+
+import time
+import psutil
+import asyncio
+import uuid
+
+from urllib.parse import urlparse
+import random
+from abc import ABC, abstractmethod
+
+
+class RateLimiter:
+    def __init__(
+        self,
+        base_delay: Tuple[float, float] = (1.0, 3.0),
+        max_delay: float = 60.0,
+        max_retries: int = 3,
+        rate_limit_codes: List[int] = None,
+    ):
+        self.base_delay = base_delay
+        self.max_delay = max_delay
+        self.max_retries = max_retries
+        self.rate_limit_codes = rate_limit_codes or [429, 503]
+        self.domains: Dict[str, DomainState] = {}
+
+    def get_domain(self, url: str) -> str:
+        return urlparse(url).netloc
+
+    async def wait_if_needed(self, url: str) -> None:
+        domain = self.get_domain(url)
+        state = self.domains.get(domain)
+
+        if not state:
+            self.domains[domain] = DomainState()
+            state = self.domains[domain]
+
+        now = time.time()
+        if state.last_request_time:
+            wait_time = max(0, state.current_delay - (now - state.last_request_time))
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+
+        # Random delay within base range if no current delay
+        if state.current_delay == 0:
+            state.current_delay = random.uniform(*self.base_delay)
+
+        state.last_request_time = time.time()
+
+    def update_delay(self, url: str, status_code: int) -> bool:
+        domain = self.get_domain(url)
+        state = self.domains[domain]
+
+        if status_code in self.rate_limit_codes:
+            state.fail_count += 1
+            if state.fail_count > self.max_retries:
+                return False
+
+            # Exponential backoff with random jitter
+            state.current_delay = min(
+                state.current_delay * 2 * random.uniform(0.75, 1.25), self.max_delay
+            )
+        else:
+            # Gradually reduce delay on success
+            state.current_delay = max(
+                random.uniform(*self.base_delay), state.current_delay * 0.75
+            )
+            state.fail_count = 0
+
+        return True
+
+
+
+class BaseDispatcher(ABC):
+    def __init__(
+        self,
+        rate_limiter: Optional[RateLimiter] = None,
+        monitor: Optional[CrawlerMonitor] = None,
+    ):
+        self.crawler = None
+        self._domain_last_hit: Dict[str, float] = {}
+        self.concurrent_sessions = 0
+        self.rate_limiter = rate_limiter
+        self.monitor = monitor
+
+    @abstractmethod
+    async def crawl_url(
+        self,
+        url: str,
+        config: CrawlerRunConfig,
+        task_id: str,
+        monitor: Optional[CrawlerMonitor] = None,
+    ) -> CrawlerTaskResult:
+        pass
+
+    @abstractmethod
+    async def run_urls(
+        self,
+        urls: List[str],
+        crawler: AsyncWebCrawler,  # noqa: F821
+        config: CrawlerRunConfig,
+        monitor: Optional[CrawlerMonitor] = None,
+    ) -> List[CrawlerTaskResult]:
+        pass
+
+
+class MemoryAdaptiveDispatcher(BaseDispatcher):
+    def __init__(
+        self,
+        memory_threshold_percent: float = 90.0,
+        critical_threshold_percent: float = 95.0,  # New critical threshold
+        recovery_threshold_percent: float = 85.0,  # New recovery threshold
+        check_interval: float = 1.0,
+        max_session_permit: int = 20,
+        fairness_timeout: float = 600.0,  # 10 minutes before prioritizing long-waiting URLs
+        rate_limiter: Optional[RateLimiter] = None,
+        monitor: Optional[CrawlerMonitor] = None,
+    ):
+        super().__init__(rate_limiter, monitor)
+        self.memory_threshold_percent = memory_threshold_percent
+        self.critical_threshold_percent = critical_threshold_percent
+        self.recovery_threshold_percent = recovery_threshold_percent
+        self.check_interval = check_interval
+        self.max_session_permit = max_session_permit
+        self.fairness_timeout = fairness_timeout
+        self.result_queue = asyncio.Queue()
+        self.task_queue = asyncio.PriorityQueue()  # Priority queue for better management
+        self.memory_pressure_mode = False  # Flag to indicate when we're in memory pressure mode
+        self.current_memory_percent = 0.0  # Track current memory usage
+        
+    async def _memory_monitor_task(self):
+        """Background task to continuously monitor memory usage and update state"""
+        while True:
+            self.current_memory_percent = psutil.virtual_memory().percent
+            
+            # Enter memory pressure mode if we cross the threshold
+            if not self.memory_pressure_mode and self.current_memory_percent >= self.memory_threshold_percent:
+                self.memory_pressure_mode = True
+                if self.monitor:
+                    self.monitor.update_memory_status("PRESSURE")
+            
+            # Exit memory pressure mode if we go below recovery threshold
+            elif self.memory_pressure_mode and self.current_memory_percent <= self.recovery_threshold_percent:
+                self.memory_pressure_mode = False
+                if self.monitor:
+                    self.monitor.update_memory_status("NORMAL")
+            
+            # In critical mode, we might need to take more drastic action
+            if self.current_memory_percent >= self.critical_threshold_percent:
+                if self.monitor:
+                    self.monitor.update_memory_status("CRITICAL")
+                # We could implement additional memory-saving measures here
+                
+            await asyncio.sleep(self.check_interval)
+    
+    def _get_priority_score(self, wait_time: float, retry_count: int) -> float:
+        """Calculate priority score (lower is higher priority)
+        - URLs waiting longer than fairness_timeout get higher priority
+        - More retry attempts decreases priority
+        """
+        if wait_time > self.fairness_timeout:
+            # High priority for long-waiting URLs
+            return -wait_time
+        # Standard priority based on retries
+        return retry_count
+    
+    async def crawl_url(
+        self,
+        url: str,
+        config: CrawlerRunConfig,
+        task_id: str,
+        retry_count: int = 0,
+    ) -> CrawlerTaskResult:
+        start_time = time.time()
+        error_message = ""
+        memory_usage = peak_memory = 0.0
+        
+        # Get starting memory for accurate measurement
+        process = psutil.Process()
+        start_memory = process.memory_info().rss / (1024 * 1024)
+        
+        try:
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id, 
+                    status=CrawlStatus.IN_PROGRESS, 
+                    start_time=start_time,
+                    retry_count=retry_count
+                )
+                
+            self.concurrent_sessions += 1
+            
+            if self.rate_limiter:
+                await self.rate_limiter.wait_if_needed(url)
+                
+            # Check if we're in critical memory state
+            if self.current_memory_percent >= self.critical_threshold_percent:
+                # Requeue this task with increased priority and retry count
+                enqueue_time = time.time()
+                priority = self._get_priority_score(enqueue_time - start_time, retry_count + 1)
+                await self.task_queue.put((priority, (url, task_id, retry_count + 1, enqueue_time)))
+                
+                # Update monitoring
+                if self.monitor:
+                    self.monitor.update_task(
+                        task_id,
+                        status=CrawlStatus.QUEUED,
+                        error_message="Requeued due to critical memory pressure"
+                    )
+                
+                # Return placeholder result with requeued status
+                return CrawlerTaskResult(
+                    task_id=task_id,
+                    url=url,
+                    result=CrawlResult(
+                        url=url, html="", metadata={"status": "requeued"}, 
+                        success=False, error_message="Requeued due to critical memory pressure"
+                    ),
+                    memory_usage=0,
+                    peak_memory=0,
+                    start_time=start_time,
+                    end_time=time.time(),
+                    error_message="Requeued due to critical memory pressure",
+                    retry_count=retry_count + 1
+                )
+            
+            # Execute the crawl
+            result = await self.crawler.arun(url, config=config, session_id=task_id)
+            
+            # Measure memory usage
+            end_memory = process.memory_info().rss / (1024 * 1024)
+            memory_usage = peak_memory = end_memory - start_memory
+            
+            # Handle rate limiting
+            if self.rate_limiter and result.status_code:
+                if not self.rate_limiter.update_delay(url, result.status_code):
+                    error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
+                    if self.monitor:
+                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                        
+            # Update status based on result
+            if not result.success:
+                error_message = result.error_message
+                if self.monitor:
+                    self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+            elif self.monitor:
+                self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
+                
+        except Exception as e:
+            error_message = str(e)
+            if self.monitor:
+                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+            result = CrawlResult(
+                url=url, html="", metadata={}, success=False, error_message=str(e)
+            )
+            
+        finally:
+            end_time = time.time()
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id,
+                    end_time=end_time,
+                    memory_usage=memory_usage,
+                    peak_memory=peak_memory,
+                    error_message=error_message,
+                    retry_count=retry_count
+                )
+            self.concurrent_sessions -= 1
+            
+        return CrawlerTaskResult(
+            task_id=task_id,
+            url=url,
+            result=result,
+            memory_usage=memory_usage,
+            peak_memory=peak_memory,
+            start_time=start_time,
+            end_time=end_time,
+            error_message=error_message,
+            retry_count=retry_count
+        )
+        
+    async def run_urls(
+        self,
+        urls: List[str],
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlerTaskResult]:
+        self.crawler = crawler
+        
+        # Start the memory monitor task
+        memory_monitor = asyncio.create_task(self._memory_monitor_task())
+        
+        if self.monitor:
+            self.monitor.start()
+            
+        results = []
+        
+        try:
+            # Initialize task queue
+            for url in urls:
+                task_id = str(uuid.uuid4())
+                if self.monitor:
+                    self.monitor.add_task(task_id, url)
+                # Add to queue with initial priority 0, retry count 0, and current time
+                await self.task_queue.put((0, (url, task_id, 0, time.time())))
+                
+            active_tasks = []
+            
+            # Process until both queues are empty
+            while not self.task_queue.empty() or active_tasks:
+                # If memory pressure is low, start new tasks
+                if not self.memory_pressure_mode and len(active_tasks) < self.max_session_permit:
+                    try:
+                        # Try to get a task with timeout to avoid blocking indefinitely
+                        priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for(
+                            self.task_queue.get(), timeout=0.1
+                        )
+                        
+                        # Create and start the task
+                        task = asyncio.create_task(
+                            self.crawl_url(url, config, task_id, retry_count)
+                        )
+                        active_tasks.append(task)
+                        
+                        # Update waiting time in monitor
+                        if self.monitor:
+                            wait_time = time.time() - enqueue_time
+                            self.monitor.update_task(
+                                task_id, 
+                                wait_time=wait_time,
+                                status=CrawlStatus.IN_PROGRESS
+                            )
+                            
+                    except asyncio.TimeoutError:
+                        # No tasks in queue, that's fine
+                        pass
+                        
+                # Wait for completion even if queue is starved
+                if active_tasks:
+                    done, pending = await asyncio.wait(
+                        active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
+                    )
+                    
+                    # Process completed tasks
+                    for completed_task in done:
+                        result = await completed_task
+                        results.append(result)
+                        
+                    # Update active tasks list
+                    active_tasks = list(pending)
+                else:
+                    # If no active tasks but still waiting, sleep briefly
+                    await asyncio.sleep(self.check_interval / 2)
+                    
+                # Update priorities for waiting tasks if needed
+                await self._update_queue_priorities()
+                
+            return results
+
+        except Exception as e:
+            if self.monitor:
+                self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")                
+        
+        finally:
+            # Clean up
+            memory_monitor.cancel()
+            if self.monitor:
+                self.monitor.stop()
+                
+    async def _update_queue_priorities(self):
+        """Periodically update priorities of items in the queue to prevent starvation"""
+        # Skip if queue is empty
+        if self.task_queue.empty():
+            return
+            
+        # Use a drain-and-refill approach to update all priorities
+        temp_items = []
+        
+        # Drain the queue (with a safety timeout to prevent blocking)
+        try:
+            drain_start = time.time()
+            while not self.task_queue.empty() and time.time() - drain_start < 5.0:  # 5 second safety timeout
+                try:
+                    # Get item from queue with timeout
+                    priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for(
+                        self.task_queue.get(), timeout=0.1
+                    )
+                    
+                    # Calculate new priority based on current wait time
+                    current_time = time.time()
+                    wait_time = current_time - enqueue_time
+                    new_priority = self._get_priority_score(wait_time, retry_count)
+                    
+                    # Store with updated priority
+                    temp_items.append((new_priority, (url, task_id, retry_count, enqueue_time)))
+                    
+                    # Update monitoring stats for this task
+                    if self.monitor and task_id in self.monitor.stats:
+                        self.monitor.update_task(task_id, wait_time=wait_time)
+                        
+                except asyncio.TimeoutError:
+                    # Queue might be empty or very slow
+                    break
+        except Exception as e:
+            # If anything goes wrong, make sure we refill the queue with what we've got
+            self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")
+        
+        # Calculate queue statistics
+        if temp_items and self.monitor:
+            total_queued = len(temp_items)
+            wait_times = [item[1][3] for item in temp_items]
+            highest_wait_time = time.time() - min(wait_times) if wait_times else 0
+            avg_wait_time = sum(time.time() - t for t in wait_times) / len(wait_times) if wait_times else 0
+            
+            # Update queue statistics in monitor
+            self.monitor.update_queue_statistics(
+                total_queued=total_queued,
+                highest_wait_time=highest_wait_time,
+                avg_wait_time=avg_wait_time
+            )
+        
+        # Sort by priority (lowest number = highest priority)
+        temp_items.sort(key=lambda x: x[0])
+        
+        # Refill the queue with updated priorities
+        for item in temp_items:
+            await self.task_queue.put(item)
+                
+    async def run_urls_stream(
+        self,
+        urls: List[str],
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlerTaskResult, None]:
+        self.crawler = crawler
+        
+        # Start the memory monitor task
+        memory_monitor = asyncio.create_task(self._memory_monitor_task())
+        
+        if self.monitor:
+            self.monitor.start()
+            
+        try:
+            # Initialize task queue
+            for url in urls:
+                task_id = str(uuid.uuid4())
+                if self.monitor:
+                    self.monitor.add_task(task_id, url)
+                # Add to queue with initial priority 0, retry count 0, and current time
+                await self.task_queue.put((0, (url, task_id, 0, time.time())))
+                
+            active_tasks = []
+            completed_count = 0
+            total_urls = len(urls)
+            
+            while completed_count < total_urls:
+                # If memory pressure is low, start new tasks
+                if not self.memory_pressure_mode and len(active_tasks) < self.max_session_permit:
+                    try:
+                        # Try to get a task with timeout
+                        priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for(
+                            self.task_queue.get(), timeout=0.1
+                        )
+                        
+                        # Create and start the task
+                        task = asyncio.create_task(
+                            self.crawl_url(url, config, task_id, retry_count)
+                        )
+                        active_tasks.append(task)
+                        
+                        # Update waiting time in monitor
+                        if self.monitor:
+                            wait_time = time.time() - enqueue_time
+                            self.monitor.update_task(
+                                task_id, 
+                                wait_time=wait_time,
+                                status=CrawlStatus.IN_PROGRESS
+                            )
+                            
+                    except asyncio.TimeoutError:
+                        # No tasks in queue, that's fine
+                        pass
+                        
+                # Process completed tasks and yield results
+                if active_tasks:
+                    done, pending = await asyncio.wait(
+                        active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
+                    )
+                    
+                    for completed_task in done:
+                        result = await completed_task
+                        
+                        # Only count as completed if it wasn't requeued
+                        if "requeued" not in result.error_message:
+                            completed_count += 1
+                            yield result
+                        
+                    # Update active tasks list
+                    active_tasks = list(pending)
+                else:
+                    # If no active tasks but still waiting, sleep briefly
+                    await asyncio.sleep(self.check_interval / 2)
+                
+                # Update priorities for waiting tasks if needed
+                await self._update_queue_priorities()
+                
+        finally:
+            # Clean up
+            memory_monitor.cancel()
+            if self.monitor:
+                self.monitor.stop()
+                
+
+class SemaphoreDispatcher(BaseDispatcher):
+    def __init__(
+        self,
+        semaphore_count: int = 5,
+        max_session_permit: int = 20,
+        rate_limiter: Optional[RateLimiter] = None,
+        monitor: Optional[CrawlerMonitor] = None,
+    ):
+        super().__init__(rate_limiter, monitor)
+        self.semaphore_count = semaphore_count
+        self.max_session_permit = max_session_permit
+
+    async def crawl_url(
+        self,
+        url: str,
+        config: CrawlerRunConfig,
+        task_id: str,
+        semaphore: asyncio.Semaphore = None,
+    ) -> CrawlerTaskResult:
+        start_time = time.time()
+        error_message = ""
+        memory_usage = peak_memory = 0.0
+
+        try:
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id, status=CrawlStatus.IN_PROGRESS, start_time=start_time
+                )
+
+            if self.rate_limiter:
+                await self.rate_limiter.wait_if_needed(url)
+
+            async with semaphore:
+                process = psutil.Process()
+                start_memory = process.memory_info().rss / (1024 * 1024)
+                result = await self.crawler.arun(url, config=config, session_id=task_id)
+                end_memory = process.memory_info().rss / (1024 * 1024)
+
+                memory_usage = peak_memory = end_memory - start_memory
+
+                if self.rate_limiter and result.status_code:
+                    if not self.rate_limiter.update_delay(url, result.status_code):
+                        error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
+                        if self.monitor:
+                            self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                        return CrawlerTaskResult(
+                            task_id=task_id,
+                            url=url,
+                            result=result,
+                            memory_usage=memory_usage,
+                            peak_memory=peak_memory,
+                            start_time=start_time,
+                            end_time=time.time(),
+                            error_message=error_message,
+                        )
+
+                if not result.success:
+                    error_message = result.error_message
+                    if self.monitor:
+                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                elif self.monitor:
+                    self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
+
+        except Exception as e:
+            error_message = str(e)
+            if self.monitor:
+                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+            result = CrawlResult(
+                url=url, html="", metadata={}, success=False, error_message=str(e)
+            )
+
+        finally:
+            end_time = time.time()
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id,
+                    end_time=end_time,
+                    memory_usage=memory_usage,
+                    peak_memory=peak_memory,
+                    error_message=error_message,
+                )
+
+        return CrawlerTaskResult(
+            task_id=task_id,
+            url=url,
+            result=result,
+            memory_usage=memory_usage,
+            peak_memory=peak_memory,
+            start_time=start_time,
+            end_time=end_time,
+            error_message=error_message,
+        )
+
+    async def run_urls(
+        self,
+        crawler: AsyncWebCrawler,  # noqa: F821
+        urls: List[str],
+        config: CrawlerRunConfig,
+    ) -> List[CrawlerTaskResult]:
+        self.crawler = crawler
+        if self.monitor:
+            self.monitor.start()
+
+        try:
+            semaphore = asyncio.Semaphore(self.semaphore_count)
+            tasks = []
+
+            for url in urls:
+                task_id = str(uuid.uuid4())
+                if self.monitor:
+                    self.monitor.add_task(task_id, url)
+                task = asyncio.create_task(
+                    self.crawl_url(url, config, task_id, semaphore)
+                )
+                tasks.append(task)
+
+            return await asyncio.gather(*tasks, return_exceptions=True)
+        finally:
+            if self.monitor:
+                self.monitor.stop()

crawl4ai/async_logger.py

@@ -0,0 +1,323 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import Optional, Dict, Any
+from colorama import Fore, Style, init
+import os
+from datetime import datetime
+
+
+class LogLevel(Enum):
+    DEBUG = 1
+    INFO = 2
+    SUCCESS = 3
+    WARNING = 4
+    ERROR = 5
+
+
+
+class AsyncLoggerBase(ABC):
+    @abstractmethod
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        pass
+
+    @abstractmethod
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        pass
+
+    @abstractmethod
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        pass
+
+    @abstractmethod
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        pass
+
+    @abstractmethod
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        pass
+
+    @abstractmethod
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+        pass
+
+    @abstractmethod
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+        pass
+
+class AsyncLogger(AsyncLoggerBase):
+    """
+    Asynchronous logger with support for colored console output and file logging.
+    Supports templated messages with colored components.
+    """
+
+    DEFAULT_ICONS = {
+        "INIT": "→",
+        "READY": "✓",
+        "FETCH": "↓",
+        "SCRAPE": "◆",
+        "EXTRACT": "■",
+        "COMPLETE": "●",
+        "ERROR": "×",
+        "DEBUG": "⋯",
+        "INFO": "ℹ",
+        "WARNING": "⚠",
+    }
+
+    DEFAULT_COLORS = {
+        LogLevel.DEBUG: Fore.LIGHTBLACK_EX,
+        LogLevel.INFO: Fore.CYAN,
+        LogLevel.SUCCESS: Fore.GREEN,
+        LogLevel.WARNING: Fore.YELLOW,
+        LogLevel.ERROR: Fore.RED,
+    }
+
+    def __init__(
+        self,
+        log_file: Optional[str] = None,
+        log_level: LogLevel = LogLevel.DEBUG,
+        tag_width: int = 10,
+        icons: Optional[Dict[str, str]] = None,
+        colors: Optional[Dict[LogLevel, str]] = None,
+        verbose: bool = True,
+    ):
+        """
+        Initialize the logger.
+
+        Args:
+            log_file: Optional file path for logging
+            log_level: Minimum log level to display
+            tag_width: Width for tag formatting
+            icons: Custom icons for different tags
+            colors: Custom colors for different log levels
+            verbose: Whether to output to console
+        """
+        init()  # Initialize colorama
+        self.log_file = log_file
+        self.log_level = log_level
+        self.tag_width = tag_width
+        self.icons = icons or self.DEFAULT_ICONS
+        self.colors = colors or self.DEFAULT_COLORS
+        self.verbose = verbose
+
+        # Create log file directory if needed
+        if log_file:
+            os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
+
+    def _format_tag(self, tag: str) -> str:
+        """Format a tag with consistent width."""
+        return f"[{tag}]".ljust(self.tag_width, ".")
+
+    def _get_icon(self, tag: str) -> str:
+        """Get the icon for a tag, defaulting to info icon if not found."""
+        return self.icons.get(tag, self.icons["INFO"])
+
+    def _write_to_file(self, message: str):
+        """Write a message to the log file if configured."""
+        if self.log_file:
+            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+            with open(self.log_file, "a", encoding="utf-8") as f:
+                # Strip ANSI color codes for file output
+                clean_message = message.replace(Fore.RESET, "").replace(
+                    Style.RESET_ALL, ""
+                )
+                for color in vars(Fore).values():
+                    if isinstance(color, str):
+                        clean_message = clean_message.replace(color, "")
+                f.write(f"[{timestamp}] {clean_message}\n")
+
+    def _log(
+        self,
+        level: LogLevel,
+        message: str,
+        tag: str,
+        params: Optional[Dict[str, Any]] = None,
+        colors: Optional[Dict[str, str]] = None,
+        base_color: Optional[str] = None,
+        **kwargs,
+    ):
+        """
+        Core logging method that handles message formatting and output.
+
+        Args:
+            level: Log level for this message
+            message: Message template string
+            tag: Tag for the message
+            params: Parameters to format into the message
+            colors: Color overrides for specific parameters
+            base_color: Base color for the entire message
+        """
+        if level.value < self.log_level.value:
+            return
+
+        # Format the message with parameters if provided
+        if params:
+            try:
+                # First format the message with raw parameters
+                formatted_message = message.format(**params)
+
+                # Then apply colors if specified
+                color_map = {
+                    "green": Fore.GREEN,
+                    "red": Fore.RED,
+                    "yellow": Fore.YELLOW,
+                    "blue": Fore.BLUE,
+                    "cyan": Fore.CYAN,
+                    "magenta": Fore.MAGENTA,
+                    "white": Fore.WHITE,
+                    "black": Fore.BLACK,
+                    "reset": Style.RESET_ALL,
+                }
+                if colors:
+                    for key, color in colors.items():
+                        # Find the formatted value in the message and wrap it with color
+                        if color in color_map:
+                            color = color_map[color]
+                        if key in params:
+                            value_str = str(params[key])
+                            formatted_message = formatted_message.replace(
+                                value_str, f"{color}{value_str}{Style.RESET_ALL}"
+                            )
+
+            except KeyError as e:
+                formatted_message = (
+                    f"LOGGING ERROR: Missing parameter {e} in message template"
+                )
+                level = LogLevel.ERROR
+        else:
+            formatted_message = message
+
+        # Construct the full log line
+        color = base_color or self.colors[level]
+        log_line = f"{color}{self._format_tag(tag)} {self._get_icon(tag)} {formatted_message}{Style.RESET_ALL}"
+
+        # Output to console if verbose
+        if self.verbose or kwargs.get("force_verbose", False):
+            print(log_line)
+
+        # Write to file if configured
+        self._write_to_file(log_line)
+
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        """Log a debug message."""
+        self._log(LogLevel.DEBUG, message, tag, **kwargs)
+
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        """Log an info message."""
+        self._log(LogLevel.INFO, message, tag, **kwargs)
+
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        """Log a success message."""
+        self._log(LogLevel.SUCCESS, message, tag, **kwargs)
+
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        """Log a warning message."""
+        self._log(LogLevel.WARNING, message, tag, **kwargs)
+
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        """Log an error message."""
+        self._log(LogLevel.ERROR, message, tag, **kwargs)
+
+    def url_status(
+        self,
+        url: str,
+        success: bool,
+        timing: float,
+        tag: str = "FETCH",
+        url_length: int = 50,
+    ):
+        """
+        Convenience method for logging URL fetch status.
+
+        Args:
+            url: The URL being processed
+            success: Whether the operation was successful
+            timing: Time taken for the operation
+            tag: Tag for the message
+            url_length: Maximum length for URL in log
+        """
+        self._log(
+            level=LogLevel.SUCCESS if success else LogLevel.ERROR,
+            message="{url:.{url_length}}... | Status: {status} | Time: {timing:.2f}s",
+            tag=tag,
+            params={
+                "url": url,
+                "url_length": url_length,
+                "status": success,
+                "timing": timing,
+            },
+            colors={
+                "status": Fore.GREEN if success else Fore.RED,
+                "timing": Fore.YELLOW,
+            },
+        )
+
+    def error_status(
+        self, url: str, error: str, tag: str = "ERROR", url_length: int = 50
+    ):
+        """
+        Convenience method for logging error status.
+
+        Args:
+            url: The URL being processed
+            error: Error message
+            tag: Tag for the message
+            url_length: Maximum length for URL in log
+        """
+        self._log(
+            level=LogLevel.ERROR,
+            message="{url:.{url_length}}... | Error: {error}",
+            tag=tag,
+            params={"url": url, "url_length": url_length, "error": error},
+        )
+
+class AsyncFileLogger(AsyncLoggerBase):
+    """
+    File-only asynchronous logger that writes logs to a specified file.
+    """
+
+    def __init__(self, log_file: str):
+        """
+        Initialize the file logger.
+
+        Args:
+            log_file: File path for logging
+        """
+        self.log_file = log_file
+        os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
+
+    def _write_to_file(self, level: str, message: str, tag: str):
+        """Write a message to the log file."""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+        with open(self.log_file, "a", encoding="utf-8") as f:
+            f.write(f"[{timestamp}] [{level}] [{tag}] {message}\n")
+
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        """Log a debug message to file."""
+        self._write_to_file("DEBUG", message, tag)
+
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        """Log an info message to file."""
+        self._write_to_file("INFO", message, tag)
+
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        """Log a success message to file."""
+        self._write_to_file("SUCCESS", message, tag)
+
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        """Log a warning message to file."""
+        self._write_to_file("WARNING", message, tag)
+
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        """Log an error message to file."""
+        self._write_to_file("ERROR", message, tag)
+
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+        """Log URL fetch status to file."""
+        status = "SUCCESS" if success else "FAILED"
+        message = f"{url[:url_length]}... | Status: {status} | Time: {timing:.2f}s"
+        self._write_to_file("URL_STATUS", message, tag)
+
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+        """Log error status to file."""
+        message = f"{url[:url_length]}... | Error: {error}"
+        self._write_to_file("ERROR", message, tag)

crawl4ai/async_webcrawler.py

@@ -0,0 +1,706 @@
+from .__version__ import __version__ as crawl4ai_version
+import os
+import sys
+import time
+from colorama import Fore
+from pathlib import Path
+from typing import Optional, List
+import json
+import asyncio
+
+# from contextlib import nullcontext, asynccontextmanager
+from contextlib import asynccontextmanager
+from .models import (
+    CrawlResult,
+    MarkdownGenerationResult,
+    DispatchResult,
+    ScrapingResult,
+    CrawlResultContainer,
+    RunManyReturn
+)
+from .async_database import async_db_manager
+from .chunking_strategy import *  # noqa: F403
+from .chunking_strategy import IdentityChunking
+from .content_filter_strategy import *  # noqa: F403
+from .extraction_strategy import *  # noqa: F403
+from .extraction_strategy import NoExtractionStrategy
+from .async_crawler_strategy import (
+    AsyncCrawlerStrategy,
+    AsyncPlaywrightCrawlerStrategy,
+    AsyncCrawlResponse,
+)
+from .cache_context import CacheMode, CacheContext
+from .markdown_generation_strategy import (
+    DefaultMarkdownGenerator,
+    MarkdownGenerationStrategy,
+)
+from .deep_crawling import DeepCrawlDecorator
+from .async_logger import AsyncLogger, AsyncLoggerBase
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from .async_dispatcher import *  # noqa: F403
+from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter
+
+from .utils import (
+    sanitize_input_encode,
+    InvalidCSSSelectorError,
+    fast_format_html,
+    create_box_message,
+    get_error_context,
+    RobotsParser,
+)
+
+
+class AsyncWebCrawler:
+    """
+    Asynchronous web crawler with flexible caching capabilities.
+
+    There are two ways to use the crawler:
+
+    1. Using context manager (recommended for simple cases):
+        ```python
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(url="https://example.com")
+        ```
+
+    2. Using explicit lifecycle management (recommended for long-running applications):
+        ```python
+        crawler = AsyncWebCrawler()
+        await crawler.start()
+
+        # Use the crawler multiple times
+        result1 = await crawler.arun(url="https://example.com")
+        result2 = await crawler.arun(url="https://another.com")
+
+        await crawler.close()
+        ```
+
+    Attributes:
+        browser_config (BrowserConfig): Configuration object for browser settings.
+        crawler_strategy (AsyncCrawlerStrategy): Strategy for crawling web pages.
+        logger (AsyncLogger): Logger instance for recording events and errors.
+        crawl4ai_folder (str): Directory for storing cache.
+        base_directory (str): Base directory for storing cache.
+        ready (bool): Whether the crawler is ready for use.
+
+    Methods:
+        start(): Start the crawler explicitly without using context manager.
+        close(): Close the crawler explicitly without using context manager.
+        arun(): Run the crawler for a single source: URL (web, local file, or raw HTML).
+        awarmup(): Perform warmup sequence.
+        arun_many(): Run the crawler for multiple sources.
+        aprocess_html(): Process HTML content.
+
+    Typical Usage:
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(url="https://example.com")
+            print(result.markdown)
+
+        Using configuration:
+        browser_config = BrowserConfig(browser_type="chromium", headless=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            crawler_config = CrawlerRunConfig(
+                cache_mode=CacheMode.BYPASS
+            )
+            result = await crawler.arun(url="https://example.com", config=crawler_config)
+            print(result.markdown)
+    """
+
+    _domain_last_hit = {}
+
+    def __init__(
+        self,
+        crawler_strategy: AsyncCrawlerStrategy = None,
+        config: BrowserConfig = None,
+        base_directory: str = str(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())),
+        thread_safe: bool = False,
+        logger: AsyncLoggerBase = None,
+        **kwargs,
+    ):
+        """
+        Initialize the AsyncWebCrawler.
+
+        Args:
+            crawler_strategy: Strategy for crawling web pages. Default AsyncPlaywrightCrawlerStrategy
+            config: Configuration object for browser settings. Default BrowserConfig()
+            base_directory: Base directory for storing cache
+            thread_safe: Whether to use thread-safe operations
+            **kwargs: Additional arguments for backwards compatibility
+        """
+        # Handle browser configuration
+        browser_config = config or BrowserConfig()
+
+        self.browser_config = browser_config
+
+        # Initialize logger first since other components may need it
+        self.logger = logger or AsyncLogger(
+            log_file=os.path.join(base_directory, ".crawl4ai", "crawler.log"),
+            verbose=self.browser_config.verbose,
+            tag_width=10,
+        )
+
+        # Initialize crawler strategy
+        params = {k: v for k, v in kwargs.items() if k in ["browser_config", "logger"]}
+        self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
+            browser_config=browser_config,
+            logger=self.logger,
+            **params,  # Pass remaining kwargs for backwards compatibility
+        )
+
+        # Thread safety setup
+        self._lock = asyncio.Lock() if thread_safe else None
+
+        # Initialize directories
+        self.crawl4ai_folder = os.path.join(base_directory, ".crawl4ai")
+        os.makedirs(self.crawl4ai_folder, exist_ok=True)
+        os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True)
+
+        # Initialize robots parser
+        self.robots_parser = RobotsParser()
+
+        self.ready = False
+
+        # Decorate arun method with deep crawling capabilities
+        self._deep_handler = DeepCrawlDecorator(self)
+        self.arun = self._deep_handler(self.arun)
+
+    async def start(self):
+        """
+        Start the crawler explicitly without using context manager.
+        This is equivalent to using 'async with' but gives more control over the lifecycle.
+        Returns:
+            AsyncWebCrawler: The initialized crawler instance
+        """
+        await self.crawler_strategy.__aenter__()
+        self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT")
+        self.ready = True
+        return self
+
+    async def close(self):
+        """
+        Close the crawler explicitly without using context manager.
+        This should be called when you're done with the crawler if you used start().
+
+        This method will:
+        1. Clean up browser resources
+        2. Close any open pages and contexts
+        """
+        await self.crawler_strategy.__aexit__(None, None, None)
+
+    async def __aenter__(self):
+        return await self.start()
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+
+    @asynccontextmanager
+    async def nullcontext(self):
+        """异步空上下文管理器"""
+        yield
+
+    async def arun(
+        self,
+        url: str,
+        config: CrawlerRunConfig = None,
+        **kwargs,
+    ) -> RunManyReturn:
+        """
+        Runs the crawler for a single source: URL (web, local file, or raw HTML).
+
+        Migration Guide:
+        Old way (deprecated):
+            result = await crawler.arun(
+                url="https://example.com",
+                word_count_threshold=200,
+                screenshot=True,
+                ...
+            )
+
+        New way (recommended):
+            config = CrawlerRunConfig(
+                word_count_threshold=200,
+                screenshot=True,
+                ...
+            )
+            result = await crawler.arun(url="https://example.com", crawler_config=config)
+
+        Args:
+            url: The URL to crawl (http://, https://, file://, or raw:)
+            crawler_config: Configuration object controlling crawl behavior
+            [other parameters maintained for backwards compatibility]
+
+        Returns:
+            CrawlResult: The result of crawling and processing
+        """
+        # Auto-start if not ready
+        if not self.ready:
+            await self.start()
+
+        config = config or CrawlerRunConfig()
+        if not isinstance(url, str) or not url:
+            raise ValueError("Invalid URL, make sure the URL is a non-empty string")
+
+        async with self._lock or self.nullcontext():
+            try:
+                self.logger.verbose = config.verbose
+
+                # Default to ENABLED if no cache mode specified
+                if config.cache_mode is None:
+                    config.cache_mode = CacheMode.ENABLED
+
+                # Create cache context
+                cache_context = CacheContext(url, config.cache_mode, False)
+
+                # Initialize processing variables
+                async_response: AsyncCrawlResponse = None
+                cached_result: CrawlResult = None
+                screenshot_data = None
+                pdf_data = None
+                extracted_content = None
+                start_time = time.perf_counter()
+
+                # Try to get cached result if appropriate
+                if cache_context.should_read():
+                    cached_result = await async_db_manager.aget_cached_url(url)
+
+                if cached_result:
+                    html = sanitize_input_encode(cached_result.html)
+                    extracted_content = sanitize_input_encode(
+                        cached_result.extracted_content or ""
+                    )
+                    extracted_content = (
+                        None
+                        if not extracted_content or extracted_content == "[]"
+                        else extracted_content
+                    )
+                    # If screenshot is requested but its not in cache, then set cache_result to None
+                    screenshot_data = cached_result.screenshot
+                    pdf_data = cached_result.pdf
+                    # if config.screenshot and not screenshot or config.pdf and not pdf:
+                    if config.screenshot and not screenshot_data:
+                        cached_result = None
+
+                    if config.pdf and not pdf_data:
+                        cached_result = None
+
+                    self.logger.url_status(
+                        url=cache_context.display_url,
+                        success=bool(html),
+                        timing=time.perf_counter() - start_time,
+                        tag="FETCH",
+                    )
+
+                # Update proxy configuration from rotation strategy if available
+                if config and config.proxy_rotation_strategy:
+                    next_proxy = await config.proxy_rotation_strategy.get_next_proxy()
+                    if next_proxy:
+                        self.logger.info(
+                            message="Switch proxy: {proxy}",
+                            tag="PROXY",
+                            params={"proxy": next_proxy.server},
+                        )
+                        config.proxy_config = next_proxy
+                        # config = config.clone(proxy_config=next_proxy)
+
+                # Fetch fresh content if needed
+                if not cached_result or not html:
+                    t1 = time.perf_counter()
+
+                    if config.user_agent:
+                        self.crawler_strategy.update_user_agent(config.user_agent)
+
+                    # Check robots.txt if enabled
+                    if config and config.check_robots_txt:
+                        if not await self.robots_parser.can_fetch(
+                            url, self.browser_config.user_agent
+                        ):
+                            return CrawlResult(
+                                url=url,
+                                html="",
+                                success=False,
+                                status_code=403,
+                                error_message="Access denied by robots.txt",
+                                response_headers={
+                                    "X-Robots-Status": "Blocked by robots.txt"
+                                },
+                            )
+
+                    ##############################
+                    # Call CrawlerStrategy.crawl #
+                    ##############################
+                    async_response = await self.crawler_strategy.crawl(
+                        url,
+                        config=config,  # Pass the entire config object
+                    )
+
+                    html = sanitize_input_encode(async_response.html)
+                    screenshot_data = async_response.screenshot
+                    pdf_data = async_response.pdf_data
+                    js_execution_result = async_response.js_execution_result
+
+                    t2 = time.perf_counter()
+                    self.logger.url_status(
+                        url=cache_context.display_url,
+                        success=bool(html),
+                        timing=t2 - t1,
+                        tag="FETCH",
+                    )
+
+                    ###############################################################
+                    # Process the HTML content, Call CrawlerStrategy.process_html #
+                    ###############################################################
+                    crawl_result: CrawlResult = await self.aprocess_html(
+                        url=url,
+                        html=html,
+                        extracted_content=extracted_content,
+                        config=config,  # Pass the config object instead of individual parameters
+                        screenshot=screenshot_data,
+                        pdf_data=pdf_data,
+                        verbose=config.verbose,
+                        is_raw_html=True if url.startswith("raw:") else False,
+                        **kwargs,
+                    )
+
+                    crawl_result.status_code = async_response.status_code
+                    crawl_result.redirected_url = async_response.redirected_url or url
+                    crawl_result.response_headers = async_response.response_headers
+                    crawl_result.downloaded_files = async_response.downloaded_files
+                    crawl_result.js_execution_result = js_execution_result
+                    crawl_result.ssl_certificate = (
+                        async_response.ssl_certificate
+                    )  # Add SSL certificate
+
+                    crawl_result.success = bool(html)
+                    crawl_result.session_id = getattr(config, "session_id", None)
+
+                    self.logger.success(
+                        message="{url:.50}... | Status: {status} | Total: {timing}",
+                        tag="COMPLETE",
+                        params={
+                            "url": cache_context.display_url,
+                            "status": crawl_result.success,
+                            "timing": f"{time.perf_counter() - start_time:.2f}s",
+                        },
+                        colors={
+                            "status": Fore.GREEN if crawl_result.success else Fore.RED,
+                            "timing": Fore.YELLOW,
+                        },
+                    )
+
+                    # Update cache if appropriate
+                    if cache_context.should_write() and not bool(cached_result):
+                        await async_db_manager.acache_url(crawl_result)
+
+                    return CrawlResultContainer(crawl_result)
+
+                else:
+                    self.logger.success(
+                        message="{url:.50}... | Status: {status} | Total: {timing}",
+                        tag="COMPLETE",
+                        params={
+                            "url": cache_context.display_url,
+                            "status": True,
+                            "timing": f"{time.perf_counter() - start_time:.2f}s",
+                        },
+                        colors={"status": Fore.GREEN, "timing": Fore.YELLOW},
+                    )
+
+                    cached_result.success = bool(html)
+                    cached_result.session_id = getattr(config, "session_id", None)
+                    cached_result.redirected_url = cached_result.redirected_url or url
+                    return CrawlResultContainer(cached_result)
+
+            except Exception as e:
+                error_context = get_error_context(sys.exc_info())
+
+                error_message = (
+                    f"Unexpected error in _crawl_web at line {error_context['line_no']} "
+                    f"in {error_context['function']} ({error_context['filename']}):\n"
+                    f"Error: {str(e)}\n\n"
+                    f"Code context:\n{error_context['code_context']}"
+                )
+
+                self.logger.error_status(
+                    url=url,
+                    error=create_box_message(error_message, type="error"),
+                    tag="ERROR",
+                )
+
+                return CrawlResultContainer(
+                    CrawlResult(
+                        url=url, html="", success=False, error_message=error_message
+                    )
+                )
+
+    async def aprocess_html(
+        self,
+        url: str,
+        html: str,
+        extracted_content: str,
+        config: CrawlerRunConfig,
+        screenshot: str,
+        pdf_data: str,
+        verbose: bool,
+        **kwargs,
+    ) -> CrawlResult:
+        """
+        Process HTML content using the provided configuration.
+
+        Args:
+            url: The URL being processed
+            html: Raw HTML content
+            extracted_content: Previously extracted content (if any)
+            config: Configuration object controlling processing behavior
+            screenshot: Screenshot data (if any)
+            pdf_data: PDF data (if any)
+            verbose: Whether to enable verbose logging
+            **kwargs: Additional parameters for backwards compatibility
+
+        Returns:
+            CrawlResult: Processed result containing extracted and formatted content
+        """
+        cleaned_html = ""
+        try:
+            _url = url if not kwargs.get("is_raw_html", False) else "Raw HTML"
+            t1 = time.perf_counter()
+
+            # Get scraping strategy and ensure it has a logger
+            scraping_strategy = config.scraping_strategy
+            if not scraping_strategy.logger:
+                scraping_strategy.logger = self.logger
+
+            # Process HTML content
+            params = config.__dict__.copy()
+            params.pop("url", None)
+            # add keys from kwargs to params that doesn't exist in params
+            params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
+
+            ################################
+            # Scraping Strategy Execution  #
+            ################################
+            result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
+
+            if result is None:
+                raise ValueError(
+                    f"Process HTML, Failed to extract content from the website: {url}"
+                )
+
+        except InvalidCSSSelectorError as e:
+            raise ValueError(str(e))
+        except Exception as e:
+            raise ValueError(
+                f"Process HTML, Failed to extract content from the website: {url}, error: {str(e)}"
+            )
+
+        # Extract results - handle both dict and ScrapingResult
+        if isinstance(result, dict):
+            cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
+            media = result.get("media", {})
+            links = result.get("links", {})
+            metadata = result.get("metadata", {})
+        else:
+            cleaned_html = sanitize_input_encode(result.cleaned_html)
+            media = result.media.model_dump()
+            links = result.links.model_dump()
+            metadata = result.metadata
+
+        ################################
+        # Generate Markdown            #
+        ################################
+        markdown_generator: Optional[MarkdownGenerationStrategy] = (
+            config.markdown_generator or DefaultMarkdownGenerator()
+        )
+
+        # Uncomment if by default we want to use PruningContentFilter
+        # if not config.content_filter and not markdown_generator.content_filter:
+        #     markdown_generator.content_filter = PruningContentFilter()
+
+        markdown_result: MarkdownGenerationResult = (
+            markdown_generator.generate_markdown(
+                cleaned_html=cleaned_html,
+                base_url=url,
+                # html2text_options=kwargs.get('html2text', {})
+            )
+        )
+
+        # Log processing completion
+        self.logger.info(
+            message="{url:.50}... | Time: {timing}s",
+            tag="SCRAPE",
+            params={
+                "url": _url,
+                "timing": int((time.perf_counter() - t1) * 1000) / 1000,
+            },
+        )
+
+        ################################
+        # Structured Content Extraction           #
+        ################################
+        if (
+            not bool(extracted_content)
+            and config.extraction_strategy
+            and not isinstance(config.extraction_strategy, NoExtractionStrategy)
+        ):
+            t1 = time.perf_counter()
+            # Choose content based on input_format
+            content_format = config.extraction_strategy.input_format
+            if content_format == "fit_markdown" and not markdown_result.fit_markdown:
+                self.logger.warning(
+                    message="Fit markdown requested but not available. Falling back to raw markdown.",
+                    tag="EXTRACT",
+                    params={"url": _url},
+                )
+                content_format = "markdown"
+
+            content = {
+                "markdown": markdown_result.raw_markdown,
+                "html": html,
+                "cleaned_html": cleaned_html,
+                "fit_markdown": markdown_result.fit_markdown,
+            }.get(content_format, markdown_result.raw_markdown)
+
+            # Use IdentityChunking for HTML input, otherwise use provided chunking strategy
+            chunking = (
+                IdentityChunking()
+                if content_format in ["html", "cleaned_html"]
+                else config.chunking_strategy
+            )
+            sections = chunking.chunk(content)
+            extracted_content = config.extraction_strategy.run(url, sections)
+            extracted_content = json.dumps(
+                extracted_content, indent=4, default=str, ensure_ascii=False
+            )
+
+            # Log extraction completion
+            self.logger.info(
+                message="Completed for {url:.50}... | Time: {timing}s",
+                tag="EXTRACT",
+                params={"url": _url, "timing": time.perf_counter() - t1},
+            )
+
+        # Handle screenshot and PDF data
+        screenshot_data = None if not screenshot else screenshot
+        pdf_data = None if not pdf_data else pdf_data
+
+        # Apply HTML formatting if requested
+        if config.prettiify:
+            cleaned_html = fast_format_html(cleaned_html)
+
+        # Return complete crawl result
+        return CrawlResult(
+            url=url,
+            html=html,
+            cleaned_html=cleaned_html,
+            markdown=markdown_result,
+            media=media,
+            links=links,
+            metadata=metadata,
+            screenshot=screenshot_data,
+            pdf=pdf_data,
+            extracted_content=extracted_content,
+            success=True,
+            error_message="",
+        )
+
+    async def arun_many(
+        self,
+        urls: List[str],
+        config: Optional[CrawlerRunConfig] = None,
+        dispatcher: Optional[BaseDispatcher] = None,
+        # Legacy parameters maintained for backwards compatibility
+        # word_count_threshold=MIN_WORD_THRESHOLD,
+        # extraction_strategy: ExtractionStrategy = None,
+        # chunking_strategy: ChunkingStrategy = RegexChunking(),
+        # content_filter: RelevantContentFilter = None,
+        # cache_mode: Optional[CacheMode] = None,
+        # bypass_cache: bool = False,
+        # css_selector: str = None,
+        # screenshot: bool = False,
+        # pdf: bool = False,
+        # user_agent: str = None,
+        # verbose=True,
+        **kwargs,
+    ) -> RunManyReturn:
+        """
+        Runs the crawler for multiple URLs concurrently using a configurable dispatcher strategy.
+
+        Args:
+        urls: List of URLs to crawl
+        config: Configuration object controlling crawl behavior for all URLs
+        dispatcher: The dispatcher strategy instance to use. Defaults to MemoryAdaptiveDispatcher
+        [other parameters maintained for backwards compatibility]
+
+        Returns:
+        Union[List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
+            Either a list of all results or an async generator yielding results
+
+        Examples:
+
+        # Batch processing (default)
+        results = await crawler.arun_many(
+            urls=["https://example1.com", "https://example2.com"],
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+        )
+        for result in results:
+            print(f"Processed {result.url}: {len(result.markdown)} chars")
+
+        # Streaming results
+        async for result in await crawler.arun_many(
+            urls=["https://example1.com", "https://example2.com"],
+            config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS, stream=True),
+        ):
+            print(f"Processed {result.url}: {len(result.markdown)} chars")
+        """
+        config = config or CrawlerRunConfig()
+        # if config is None:
+        #     config = CrawlerRunConfig(
+        #         word_count_threshold=word_count_threshold,
+        #         extraction_strategy=extraction_strategy,
+        #         chunking_strategy=chunking_strategy,
+        #         content_filter=content_filter,
+        #         cache_mode=cache_mode,
+        #         bypass_cache=bypass_cache,
+        #         css_selector=css_selector,
+        #         screenshot=screenshot,
+        #         pdf=pdf,
+        #         verbose=verbose,
+        #         **kwargs,
+        #     )
+
+        if dispatcher is None:
+            dispatcher = MemoryAdaptiveDispatcher(
+                rate_limiter=RateLimiter(
+                    base_delay=(1.0, 3.0), max_delay=60.0, max_retries=3
+                ),
+            )
+
+        def transform_result(task_result):
+            return (
+                setattr(
+                    task_result.result,
+                    "dispatch_result",
+                    DispatchResult(
+                        task_id=task_result.task_id,
+                        memory_usage=task_result.memory_usage,
+                        peak_memory=task_result.peak_memory,
+                        start_time=task_result.start_time,
+                        end_time=task_result.end_time,
+                        error_message=task_result.error_message,
+                    ),
+                )
+                or task_result.result
+            )
+
+        stream = config.stream
+
+        if stream:
+
+            async def result_transformer():
+                async for task_result in dispatcher.run_urls_stream(
+                    crawler=self, urls=urls, config=config
+                ):
+                    yield transform_result(task_result)
+
+            return result_transformer()
+        else:
+            _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config)
+            return [transform_result(res) for res in _results]

crawl4ai/browser/__init__.py

@@ -0,0 +1,23 @@
+"""Browser management module for Crawl4AI.
+
+This module provides browser management capabilities using different strategies
+for browser creation and interaction.
+"""
+
+from .manager import BrowserManager
+from .profiles import BrowserProfileManager
+from .models import DockerConfig
+from .docker_registry import DockerRegistry
+from .docker_utils import DockerUtils
+from .browser_hub import BrowserHub
+from .strategies import (
+    BaseBrowserStrategy,
+    PlaywrightBrowserStrategy,
+    CDPBrowserStrategy,
+    BuiltinBrowserStrategy,
+    DockerBrowserStrategy
+)
+
+__all__ = ['BrowserManager', 'BrowserProfileManager', 'DockerConfig', 'DockerRegistry', 'DockerUtils', 'BaseBrowserStrategy',
+           'PlaywrightBrowserStrategy', 'CDPBrowserStrategy', 'BuiltinBrowserStrategy',
+           'DockerBrowserStrategy', 'BrowserHub']

crawl4ai/browser/browser_hub.py

@@ -0,0 +1,184 @@
+# browser_hub_manager.py
+import hashlib
+import json
+import asyncio
+from typing import Dict, Optional, List, Tuple
+from .manager import BrowserManager, UnavailableBehavior
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+from ..async_logger import AsyncLogger
+
+class BrowserHub:
+    """
+    Manages Browser-Hub instances for sharing across multiple pipelines.
+    
+    This class provides centralized management for browser resources, allowing
+    multiple pipelines to share browser instances efficiently, connect to
+    existing browser hubs, or create new ones with custom configurations.
+    """
+    _instances: Dict[str, BrowserManager] = {}
+    _lock = asyncio.Lock()
+    
+    @classmethod
+    async def get_browser_manager(
+        cls, 
+        config: Optional[BrowserConfig] = None,
+        hub_id: Optional[str] = None,
+        connection_info: Optional[str] = None,
+        logger: Optional[AsyncLogger] = None,
+        max_browsers_per_config: int = 10,
+        max_pages_per_browser: int = 5,
+        initial_pool_size: int = 1,
+        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
+    ) -> BrowserManager:
+        """
+        Get an existing BrowserManager or create a new one based on parameters.
+        
+        Args:
+            config: Browser configuration for new hub
+            hub_id: Identifier for the hub instance
+            connection_info: Connection string for existing hub
+            logger: Logger for recording events and errors
+            max_browsers_per_config: Maximum browsers per configuration
+            max_pages_per_browser: Maximum pages per browser
+            initial_pool_size: Initial number of browsers to create
+            page_configs: Optional configurations for pre-warming pages
+            
+        Returns:
+            BrowserManager: The requested browser manager instance
+        """
+        async with cls._lock:
+            # Scenario 3: Use existing hub via connection info
+            if connection_info:
+                instance_key = f"connection:{connection_info}"
+                if instance_key not in cls._instances:
+                    cls._instances[instance_key] = await cls._connect_to_browser_hub(
+                        connection_info, logger
+                    )
+                return cls._instances[instance_key]
+                
+            # Scenario 2: Custom configured hub
+            if config:
+                config_hash = cls._hash_config(config)
+                instance_key = hub_id or f"config:{config_hash}"
+                if instance_key not in cls._instances:
+                    cls._instances[instance_key] = await cls._create_browser_manager(
+                        config, 
+                        logger,
+                        max_browsers_per_config,
+                        max_pages_per_browser,
+                        initial_pool_size,
+                        page_configs
+                    )
+                return cls._instances[instance_key]
+            
+            # Scenario 1: Default hub
+            instance_key = "default"
+            if instance_key not in cls._instances:
+                cls._instances[instance_key] = await cls._create_default_browser_hub(
+                    logger,
+                    max_browsers_per_config,
+                    max_pages_per_browser,
+                    initial_pool_size
+                )
+            return cls._instances[instance_key]
+    
+    @classmethod
+    async def _create_browser_manager(
+        cls, 
+        config: BrowserConfig,
+        logger: Optional[AsyncLogger],
+        max_browsers_per_config: int,
+        max_pages_per_browser: int,
+        initial_pool_size: int,
+        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
+    ) -> BrowserManager:
+        """Create a new browser hub with the specified configuration."""
+        manager = BrowserManager(
+            browser_config=config,
+            logger=logger,
+            unavailable_behavior=UnavailableBehavior.ON_DEMAND,
+            max_browsers_per_config=max_browsers_per_config,
+            max_pages_per_browser=max_pages_per_browser,
+        )
+        
+        # Initialize the pool
+        await manager.initialize_pool(
+            browser_configs=[config] if config else None,
+            browsers_per_config=initial_pool_size,
+            page_configs=page_configs
+        )
+        
+        return manager
+    
+    @classmethod
+    async def _create_default_browser_hub(
+        cls,
+        logger: Optional[AsyncLogger],
+        max_browsers_per_config: int,
+        max_pages_per_browser: int,
+        initial_pool_size: int
+    ) -> BrowserManager:
+        """Create a default browser hub with standard settings."""
+        config = BrowserConfig(headless=True)
+        return await cls._create_browser_manager(
+            config, 
+            logger, 
+            max_browsers_per_config, 
+            max_pages_per_browser, 
+            initial_pool_size,
+            None
+        )
+    
+    @classmethod
+    async def _connect_to_browser_hub(
+        cls, 
+        connection_info: str,
+        logger: Optional[AsyncLogger]
+    ) -> BrowserManager:
+        """
+        Connect to an existing browser hub.
+        
+        Note: This is a placeholder for future remote connection functionality.
+        Currently creates a local instance.
+        """
+        if logger:
+            logger.info(
+                message="Remote browser hub connections not yet implemented. Creating local instance.",
+                tag="BROWSER_HUB"
+            )
+        # For now, create a default local instance
+        return await cls._create_default_browser_hub(
+            logger, 
+            max_browsers_per_config=10, 
+            max_pages_per_browser=5, 
+            initial_pool_size=1
+        )
+    
+    @classmethod
+    def _hash_config(cls, config: BrowserConfig) -> str:
+        """Create a hash of the browser configuration for identification."""
+        # Convert config to dictionary, excluding any callable objects
+        config_dict = config.__dict__.copy()
+        for key in list(config_dict.keys()):
+            if callable(config_dict[key]):
+                del config_dict[key]
+        
+        # Convert to canonical JSON string
+        config_json = json.dumps(config_dict, sort_keys=True, default=str)
+        
+        # Hash the JSON
+        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
+        return config_hash
+    
+    @classmethod
+    async def shutdown_all(cls):
+        """Close all browser hub instances and clear the registry."""
+        async with cls._lock:
+            shutdown_tasks = []
+            for hub in cls._instances.values():
+                shutdown_tasks.append(hub.close())
+            
+            if shutdown_tasks:
+                await asyncio.gather(*shutdown_tasks)
+            
+            cls._instances.clear()

crawl4ai/browser/docker/alpine/connect.Dockerfile

@@ -0,0 +1,34 @@
+# ---------- Dockerfile ----------
+    FROM alpine:latest
+
+    # Combine everything in one RUN to keep layers minimal.
+    RUN apk update && apk upgrade && \
+        apk add --no-cache \
+            chromium \
+            nss \
+            freetype \
+            harfbuzz \
+            ca-certificates \
+            ttf-freefont \
+            socat \
+            curl && \
+        addgroup -S chromium && adduser -S chromium -G chromium && \
+        mkdir -p /data && chown chromium:chromium /data && \
+        rm -rf /var/cache/apk/*
+    
+    # Copy start script, then chown/chmod in one step
+    COPY start.sh /home/chromium/start.sh
+    RUN chown chromium:chromium /home/chromium/start.sh && \
+        chmod +x /home/chromium/start.sh
+    
+    USER chromium
+    WORKDIR /home/chromium
+    
+    # Expose port used by socat (mapping 9222→9223 or whichever you prefer)
+    EXPOSE 9223
+    
+    # Simple healthcheck: is the remote debug endpoint responding?
+    HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -f http://localhost:9222/json/version || exit 1
+    
+    CMD ["./start.sh"]
+    

crawl4ai/browser/docker/alpine/launch.Dockerfile

@@ -0,0 +1,27 @@
+# ---------- Dockerfile (Idle Version) ----------
+    FROM alpine:latest
+
+    # Install only Chromium and its dependencies in a single layer
+    RUN apk update && apk upgrade && \
+        apk add --no-cache \
+            chromium \
+            nss \
+            freetype \
+            harfbuzz \
+            ca-certificates \
+            ttf-freefont \
+            socat \
+            curl && \
+        addgroup -S chromium && adduser -S chromium -G chromium && \
+        mkdir -p /data && chown chromium:chromium /data && \
+        rm -rf /var/cache/apk/*
+    
+    ENV PATH="/usr/bin:/bin:/usr/sbin:/sbin"
+
+    # Switch to a non-root user for security
+    USER chromium
+    WORKDIR /home/chromium
+    
+    # Idle: container does nothing except stay alive
+    CMD ["tail", "-f", "/dev/null"]
+    

crawl4ai/browser/docker/debian/connect.Dockerfile

@@ -0,0 +1,23 @@
+# Use Debian 12 (Bookworm) slim for a small, stable base image
+FROM debian:bookworm-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Chromium, socat, and basic fonts
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    chromium \
+    wget \
+    curl \
+    socat \
+    fonts-freefont-ttf \
+    fonts-noto-color-emoji && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Copy start.sh and make it executable
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Expose socat port (use host mapping, e.g. -p 9225:9223)
+EXPOSE 9223
+
+ENTRYPOINT ["/start.sh"]

crawl4ai/browser/docker_registry.py

@@ -0,0 +1,264 @@
+"""Docker registry module for Crawl4AI.
+
+This module provides a registry system for tracking and reusing Docker containers
+across browser sessions, improving performance and resource utilization.
+"""
+
+import os
+import json
+import time
+from typing import Dict, Optional
+
+from ..utils import get_home_folder
+
+
+class DockerRegistry:
+    """Manages a registry of Docker containers used for browser automation.
+    
+    This registry tracks containers by configuration hash, allowing reuse of appropriately
+    configured containers instead of creating new ones for each session.
+    
+    Attributes:
+        registry_file (str): Path to the registry file
+        containers (dict): Dictionary of container information
+        port_map (dict): Map of host ports to container IDs
+        last_port (int): Last port assigned
+    """
+    
+    def __init__(self, registry_file: Optional[str] = None):
+        """Initialize the registry with an optional path to the registry file.
+        
+        Args:
+            registry_file: Path to the registry file. If None, uses default path.
+        """
+        # Use the same file path as BuiltinBrowserStrategy by default
+        self.registry_file = registry_file or os.path.join(get_home_folder(), "builtin-browser", "browser_config.json")
+        self.containers = {}  # Still maintain this for backward compatibility
+        self.port_map = {}    # Will be populated from the shared file
+        self.last_port = 9222
+        self.load()
+    
+    def load(self):
+        """Load container registry from file."""
+        if os.path.exists(self.registry_file):
+            try:
+                with open(self.registry_file, 'r') as f:
+                    registry_data = json.load(f)
+                    
+                    # Initialize port_map if not present
+                    if "port_map" not in registry_data:
+                        registry_data["port_map"] = {}
+                    
+                    self.port_map = registry_data.get("port_map", {})
+                    
+                    # Extract container information from port_map entries of type "docker"
+                    self.containers = {}
+                    for port_str, browser_info in self.port_map.items():
+                        if browser_info.get("browser_type") == "docker" and "container_id" in browser_info:
+                            container_id = browser_info["container_id"]
+                            self.containers[container_id] = {
+                                "host_port": int(port_str),
+                                "config_hash": browser_info.get("config_hash", ""),
+                                "created_at": browser_info.get("created_at", time.time())
+                            }
+                    
+                    # Get last port if available
+                    if "last_port" in registry_data:
+                        self.last_port = registry_data["last_port"]
+                    else:
+                        # Find highest port in port_map
+                        ports = [int(p) for p in self.port_map.keys() if p.isdigit()]
+                        self.last_port = max(ports + [9222])
+                    
+            except Exception as e:
+                # Reset to defaults on error
+                print(f"Error loading registry: {e}")
+                self.containers = {}
+                self.port_map = {}
+                self.last_port = 9222
+        else:
+            # Initialize with defaults if file doesn't exist
+            self.containers = {}
+            self.port_map = {}
+            self.last_port = 9222
+    
+    def save(self):
+        """Save container registry to file."""
+        # First load the current file to avoid overwriting other browser types
+        current_data = {"port_map": {}, "last_port": self.last_port}
+        if os.path.exists(self.registry_file):
+            try:
+                with open(self.registry_file, 'r') as f:
+                    current_data = json.load(f)
+            except Exception:
+                pass
+        
+        # Create a new port_map dictionary
+        updated_port_map = {}
+        
+        # First, copy all non-docker entries from the existing port_map
+        for port_str, browser_info in current_data.get("port_map", {}).items():
+            if browser_info.get("browser_type") != "docker":
+                updated_port_map[port_str] = browser_info
+        
+        # Then add all current docker container entries
+        for container_id, container_info in self.containers.items():
+            port_str = str(container_info["host_port"])
+            updated_port_map[port_str] = {
+                "browser_type": "docker",
+                "container_id": container_id,
+                "cdp_url": f"http://localhost:{port_str}",
+                "config_hash": container_info["config_hash"],
+                "created_at": container_info["created_at"]
+            }
+        
+        # Replace the port_map with our updated version
+        current_data["port_map"] = updated_port_map
+        
+        # Update last_port
+        current_data["last_port"] = self.last_port
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(self.registry_file), exist_ok=True)
+        
+        # Save the updated data
+        with open(self.registry_file, 'w') as f:
+            json.dump(current_data, f, indent=2)
+    
+    def register_container(self, container_id: str, host_port: int, config_hash: str, cdp_json_config: Optional[str] = None):
+        """Register a container with its configuration hash and port mapping.
+        
+        Args:
+            container_id: Docker container ID
+            host_port: Host port mapped to container
+            config_hash: Hash of configuration used to create container
+            cdp_json_config: CDP JSON configuration if available
+        """
+        self.containers[container_id] = {
+            "host_port": host_port,
+            "config_hash": config_hash,
+            "created_at": time.time()
+        }
+        
+        # Update port_map to maintain compatibility with BuiltinBrowserStrategy
+        port_str = str(host_port)
+        self.port_map[port_str] = {
+            "browser_type": "docker",
+            "container_id": container_id,
+            "cdp_url": f"http://localhost:{port_str}",
+            "config_hash": config_hash,
+            "created_at": time.time()
+        }
+        
+        if cdp_json_config:
+            self.port_map[port_str]["cdp_json_config"] = cdp_json_config
+        
+        self.save()
+    
+    def unregister_container(self, container_id: str):
+        """Unregister a container.
+        
+        Args:
+            container_id: Docker container ID to unregister
+        """
+        if container_id in self.containers:
+            host_port = self.containers[container_id]["host_port"]
+            port_str = str(host_port)
+            
+            # Remove from port_map
+            if port_str in self.port_map:
+                del self.port_map[port_str]
+                
+            # Remove from containers
+            del self.containers[container_id]
+            
+            self.save()
+    
+    async def find_container_by_config(self, config_hash: str, docker_utils) -> Optional[str]:
+        """Find a container that matches the given configuration hash.
+        
+        Args:
+            config_hash: Hash of configuration to match
+            docker_utils: DockerUtils instance to check running containers
+            
+        Returns:
+            Container ID if found, None otherwise
+        """
+        # Search through port_map for entries with matching config_hash
+        for port_str, browser_info in self.port_map.items():
+            if (browser_info.get("browser_type") == "docker" and 
+                browser_info.get("config_hash") == config_hash and 
+                "container_id" in browser_info):
+                
+                container_id = browser_info["container_id"]
+                if await docker_utils.is_container_running(container_id):
+                    return container_id
+        
+        return None
+    
+    def get_container_host_port(self, container_id: str) -> Optional[int]:
+        """Get the host port mapped to the container.
+        
+        Args:
+            container_id: Docker container ID
+            
+        Returns:
+            Host port if container is registered, None otherwise
+        """
+        if container_id in self.containers:
+            return self.containers[container_id]["host_port"]
+        return None
+    
+    def get_next_available_port(self, docker_utils) -> int:
+        """Get the next available host port for Docker mapping.
+        
+        Args:
+            docker_utils: DockerUtils instance to check port availability
+            
+        Returns:
+            Available port number
+        """
+        # Start from last port + 1
+        port = self.last_port + 1
+        
+        # Check if port is in use (either in our registry or system-wide)
+        while str(port) in self.port_map or docker_utils.is_port_in_use(port):
+            port += 1
+        
+        # Update last port
+        self.last_port = port
+        self.save()
+        
+        return port
+    
+    def get_container_config_hash(self, container_id: str) -> Optional[str]:
+        """Get the configuration hash for a container.
+        
+        Args:
+            container_id: Docker container ID
+            
+        Returns:
+            Configuration hash if container is registered, None otherwise
+        """
+        if container_id in self.containers:
+            return self.containers[container_id]["config_hash"]
+        return None
+    
+    def cleanup_stale_containers(self, docker_utils):
+        """Clean up containers that are no longer running.
+        
+        Args:
+            docker_utils: DockerUtils instance to check container status
+        """
+        to_remove = []
+        
+        # Find containers that are no longer running
+        for port_str, browser_info in self.port_map.items():
+            if browser_info.get("browser_type") == "docker" and "container_id" in browser_info:
+                container_id = browser_info["container_id"]
+                if not docker_utils.is_container_running(container_id):
+                    to_remove.append(container_id)
+        
+        # Remove stale containers
+        for container_id in to_remove:
+            self.unregister_container(container_id)

crawl4ai/browser/docker_utils.py

@@ -0,0 +1,661 @@
+import os
+import json
+import asyncio
+import hashlib
+import tempfile
+import shutil
+import socket
+import subprocess
+from typing import Dict, List, Optional, Tuple, Union
+
+
+class DockerUtils:
+    """Utility class for Docker operations in browser automation.
+
+    This class provides methods for managing Docker images, containers,
+    and related operations needed for browser automation. It handles
+    image building, container lifecycle, port management, and registry operations.
+
+    Attributes:
+        DOCKER_FOLDER (str): Path to folder containing Docker files
+        DOCKER_CONNECT_FILE (str): Path to Dockerfile for connect mode
+        DOCKER_LAUNCH_FILE (str): Path to Dockerfile for launch mode
+        DOCKER_START_SCRIPT (str): Path to startup script for connect mode
+        DEFAULT_CONNECT_IMAGE (str): Default image name for connect mode
+        DEFAULT_LAUNCH_IMAGE (str): Default image name for launch mode
+        logger: Optional logger instance
+    """
+
+    # File paths for Docker resources
+    DOCKER_FOLDER = os.path.join(os.path.dirname(__file__), "docker")
+    DOCKER_CONNECT_FILE = os.path.join(DOCKER_FOLDER, "connect.Dockerfile")
+    DOCKER_LAUNCH_FILE = os.path.join(DOCKER_FOLDER, "launch.Dockerfile")
+    DOCKER_START_SCRIPT = os.path.join(DOCKER_FOLDER, "start.sh")
+
+    # Default image names
+    DEFAULT_CONNECT_IMAGE = "crawl4ai/browser-connect:latest"
+    DEFAULT_LAUNCH_IMAGE = "crawl4ai/browser-launch:latest"
+
+    def __init__(self, logger=None):
+        """Initialize Docker utilities.
+
+        Args:
+            logger: Optional logger for recording operations
+        """
+        self.logger = logger
+
+    # Image Management Methods
+
+    async def check_image_exists(self, image_name: str) -> bool:
+        """Check if a Docker image exists.
+
+        Args:
+            image_name: Name of the Docker image to check
+
+        Returns:
+            bool: True if the image exists, False otherwise
+        """
+        cmd = ["docker", "image", "inspect", image_name]
+
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            _, _ = await process.communicate()
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.debug(
+                    f"Error checking if image exists: {str(e)}", tag="DOCKER"
+                )
+            return False
+
+    async def build_docker_image(
+        self,
+        image_name: str,
+        dockerfile_path: str,
+        files_to_copy: Dict[str, str] = None,
+    ) -> bool:
+        """Build a Docker image from a Dockerfile.
+
+        Args:
+            image_name: Name to give the built image
+            dockerfile_path: Path to the Dockerfile
+            files_to_copy: Dict of {dest_name: source_path} for files to copy to build context
+
+        Returns:
+            bool: True if image was built successfully, False otherwise
+        """
+        # Create a temporary build context
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Copy the Dockerfile
+            shutil.copy(dockerfile_path, os.path.join(temp_dir, "Dockerfile"))
+
+            # Copy any additional files needed
+            if files_to_copy:
+                for dest_name, source_path in files_to_copy.items():
+                    shutil.copy(source_path, os.path.join(temp_dir, dest_name))
+
+            # Build the image
+            cmd = ["docker", "build", "-t", image_name, temp_dir]
+
+            if self.logger:
+                self.logger.debug(
+                    f"Building Docker image with command: {' '.join(cmd)}", tag="DOCKER"
+                )
+
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            if process.returncode != 0:
+                if self.logger:
+                    self.logger.error(
+                        message="Failed to build Docker image: {error}",
+                        tag="DOCKER",
+                        params={"error": stderr.decode()},
+                    )
+                return False
+
+            if self.logger:
+                self.logger.success(
+                    f"Successfully built Docker image: {image_name}", tag="DOCKER"
+                )
+            return True
+
+    async def ensure_docker_image_exists(
+        self, image_name: str, mode: str = "connect"
+    ) -> str:
+        """Ensure the required Docker image exists, creating it if necessary.
+
+        Args:
+            image_name: Name of the Docker image
+            mode: Either "connect" or "launch" to determine which image to build
+
+        Returns:
+            str: Name of the available Docker image
+
+        Raises:
+            Exception: If image doesn't exist and can't be built
+        """
+        # If image name is not specified, use default based on mode
+        if not image_name:
+            image_name = (
+                self.DEFAULT_CONNECT_IMAGE
+                if mode == "connect"
+                else self.DEFAULT_LAUNCH_IMAGE
+            )
+
+        # Check if the image already exists
+        if await self.check_image_exists(image_name):
+            if self.logger:
+                self.logger.debug(
+                    f"Docker image {image_name} already exists", tag="DOCKER"
+                )
+            return image_name
+
+        # If we're using a custom image that doesn't exist, warn and fail
+        if (
+            image_name != self.DEFAULT_CONNECT_IMAGE
+            and image_name != self.DEFAULT_LAUNCH_IMAGE
+        ):
+            if self.logger:
+                self.logger.warning(
+                    f"Custom Docker image {image_name} not found and cannot be automatically created",
+                    tag="DOCKER",
+                )
+            raise Exception(f"Docker image {image_name} not found")
+
+        # Build the appropriate default image
+        if self.logger:
+            self.logger.info(
+                f"Docker image {image_name} not found, creating it now...", tag="DOCKER"
+            )
+
+        if mode == "connect":
+            success = await self.build_docker_image(
+                image_name,
+                self.DOCKER_CONNECT_FILE,
+                {"start.sh": self.DOCKER_START_SCRIPT},
+            )
+        else:
+            success = await self.build_docker_image(image_name, self.DOCKER_LAUNCH_FILE)
+
+        if not success:
+            raise Exception(f"Failed to create Docker image {image_name}")
+
+        return image_name
+
+    # Container Management Methods
+
+    async def create_container(
+        self,
+        image_name: str,
+        host_port: int,
+        container_name: Optional[str] = None,
+        volumes: List[str] = None,
+        network: Optional[str] = None,
+        env_vars: Dict[str, str] = None,
+        cpu_limit: float = 1.0,
+        memory_limit: str = "1.5g",
+        extra_args: List[str] = None,
+    ) -> Optional[str]:
+        """Create a new Docker container.
+
+        Args:
+            image_name: Docker image to use
+            host_port: Port on host to map to container port 9223
+            container_name: Optional name for the container
+            volumes: List of volume mappings (e.g., ["host_path:container_path"])
+            network: Optional Docker network to use
+            env_vars: Dictionary of environment variables
+            cpu_limit: CPU limit for the container
+            memory_limit: Memory limit for the container
+            extra_args: Additional docker run arguments
+
+        Returns:
+            str: Container ID if successful, None otherwise
+        """
+        # Prepare container command
+        cmd = [
+            "docker",
+            "run",
+            "--detach",
+        ]
+
+        # Add container name if specified
+        if container_name:
+            cmd.extend(["--name", container_name])
+
+        # Add port mapping
+        cmd.extend(["-p", f"{host_port}:9223"])
+
+        # Add volumes
+        if volumes:
+            for volume in volumes:
+                cmd.extend(["-v", volume])
+
+        # Add network if specified
+        if network:
+            cmd.extend(["--network", network])
+
+        # Add environment variables
+        if env_vars:
+            for key, value in env_vars.items():
+                cmd.extend(["-e", f"{key}={value}"])
+
+        # Add CPU and memory limits
+        if cpu_limit:
+            cmd.extend(["--cpus", str(cpu_limit)])
+        if memory_limit:
+            cmd.extend(["--memory", memory_limit])
+            cmd.extend(["--memory-swap", memory_limit])
+        if self.logger:
+            self.logger.debug(
+                f"Setting CPU limit: {cpu_limit}, Memory limit: {memory_limit}",
+                tag="DOCKER",
+            )
+
+        # Add extra args
+        if extra_args:
+            cmd.extend(extra_args)
+
+        # Add image
+        cmd.append(image_name)
+
+        if self.logger:
+            self.logger.debug(
+                f"Creating Docker container with command: {' '.join(cmd)}", tag="DOCKER"
+            )
+
+        # Run docker command
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            if process.returncode != 0:
+                if self.logger:
+                    self.logger.error(
+                        message="Failed to create Docker container: {error}",
+                        tag="DOCKER",
+                        params={"error": stderr.decode()},
+                    )
+                return None
+
+            # Get container ID
+            container_id = stdout.decode().strip()
+
+            if self.logger:
+                self.logger.success(
+                    f"Created Docker container: {container_id[:12]}", tag="DOCKER"
+                )
+
+            return container_id
+
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    message="Error creating Docker container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)},
+                )
+            return None
+
+    async def is_container_running(self, container_id: str) -> bool:
+        """Check if a container is running.
+
+        Args:
+            container_id: ID of the container to check
+
+        Returns:
+            bool: True if the container is running, False otherwise
+        """
+        cmd = ["docker", "inspect", "--format", "{{.State.Running}}", container_id]
+
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, _ = await process.communicate()
+
+            return process.returncode == 0 and stdout.decode().strip() == "true"
+        except Exception as e:
+            if self.logger:
+                self.logger.debug(
+                    f"Error checking if container is running: {str(e)}", tag="DOCKER"
+                )
+            return False
+
+    async def wait_for_container_ready(
+        self, container_id: str, timeout: int = 30
+    ) -> bool:
+        """Wait for the container to be in running state.
+
+        Args:
+            container_id: ID of the container to wait for
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            bool: True if container is ready, False if timeout occurred
+        """
+        for _ in range(timeout):
+            if await self.is_container_running(container_id):
+                return True
+            await asyncio.sleep(1)
+
+        if self.logger:
+            self.logger.warning(
+                f"Container {container_id[:12]} not ready after {timeout}s timeout",
+                tag="DOCKER",
+            )
+        return False
+
+    async def stop_container(self, container_id: str) -> bool:
+        """Stop a Docker container.
+
+        Args:
+            container_id: ID of the container to stop
+
+        Returns:
+            bool: True if stopped successfully, False otherwise
+        """
+        cmd = ["docker", "stop", container_id]
+
+        try:
+            process = await asyncio.create_subprocess_exec(*cmd)
+            await process.communicate()
+
+            if self.logger:
+                self.logger.debug(
+                    f"Stopped container: {container_id[:12]}", tag="DOCKER"
+                )
+
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to stop container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)},
+                )
+            return False
+
+    async def remove_container(self, container_id: str, force: bool = True) -> bool:
+        """Remove a Docker container.
+
+        Args:
+            container_id: ID of the container to remove
+            force: Whether to force removal
+
+        Returns:
+            bool: True if removed successfully, False otherwise
+        """
+        cmd = ["docker", "rm"]
+        if force:
+            cmd.append("-f")
+        cmd.append(container_id)
+
+        try:
+            process = await asyncio.create_subprocess_exec(*cmd)
+            await process.communicate()
+
+            if self.logger:
+                self.logger.debug(
+                    f"Removed container: {container_id[:12]}", tag="DOCKER"
+                )
+
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to remove container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)},
+                )
+            return False
+
+    # Container Command Execution Methods
+
+    async def exec_in_container(
+        self, container_id: str, command: List[str], detach: bool = False
+    ) -> Tuple[int, str, str]:
+        """Execute a command in a running container.
+
+        Args:
+            container_id: ID of the container
+            command: Command to execute as a list of strings
+            detach: Whether to run the command in detached mode
+
+        Returns:
+            Tuple of (return_code, stdout, stderr)
+        """
+        cmd = ["docker", "exec"]
+        if detach:
+            cmd.append("-d")
+        cmd.append(container_id)
+        cmd.extend(command)
+
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            return process.returncode, stdout.decode(), stderr.decode()
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    message="Error executing command in container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)},
+                )
+            return -1, "", str(e)
+
+    async def start_socat_in_container(self, container_id: str) -> bool:
+        """Start socat in the container to map port 9222 to 9223.
+
+        Args:
+            container_id: ID of the container
+
+        Returns:
+            bool: True if socat started successfully, False otherwise
+        """
+        # Command to run socat as a background process
+        cmd = ["socat", "TCP-LISTEN:9223,fork", "TCP:localhost:9222"]
+
+        returncode, _, stderr = await self.exec_in_container(
+            container_id, cmd, detach=True
+        )
+
+        if returncode != 0:
+            if self.logger:
+                self.logger.error(
+                    message="Failed to start socat in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr},
+                )
+            return False
+
+        if self.logger:
+            self.logger.debug(
+                f"Started socat in container: {container_id[:12]}", tag="DOCKER"
+            )
+
+        # Wait a moment for socat to start
+        await asyncio.sleep(1)
+        return True
+
+    async def launch_chrome_in_container(
+        self, container_id: str, browser_args: List[str]
+    ) -> bool:
+        """Launch Chrome inside the container with specified arguments.
+
+        Args:
+            container_id: ID of the container
+            browser_args: Chrome command line arguments
+
+        Returns:
+            bool: True if Chrome started successfully, False otherwise
+        """
+        # Build Chrome command
+        chrome_cmd = ["chromium"]
+        chrome_cmd.extend(browser_args)
+
+        returncode, _, stderr = await self.exec_in_container(
+            container_id, chrome_cmd, detach=True
+        )
+
+        if returncode != 0:
+            if self.logger:
+                self.logger.error(
+                    message="Failed to launch Chrome in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr},
+                )
+            return False
+
+        if self.logger:
+            self.logger.debug(
+                f"Launched Chrome in container: {container_id[:12]}", tag="DOCKER"
+            )
+
+        return True
+
+    async def get_process_id_in_container(
+        self, container_id: str, process_name: str
+    ) -> Optional[int]:
+        """Get the process ID for a process in the container.
+
+        Args:
+            container_id: ID of the container
+            process_name: Name pattern to search for
+
+        Returns:
+            int: Process ID if found, None otherwise
+        """
+        cmd = ["pgrep", "-f", process_name]
+
+        returncode, stdout, _ = await self.exec_in_container(container_id, cmd)
+
+        if returncode == 0 and stdout.strip():
+            pid = int(stdout.strip().split("\n")[0])
+            return pid
+
+        return None
+
+    async def stop_process_in_container(self, container_id: str, pid: int) -> bool:
+        """Stop a process in the container by PID.
+
+        Args:
+            container_id: ID of the container
+            pid: Process ID to stop
+
+        Returns:
+            bool: True if process was stopped, False otherwise
+        """
+        cmd = ["kill", "-TERM", str(pid)]
+
+        returncode, _, stderr = await self.exec_in_container(container_id, cmd)
+
+        if returncode != 0:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to stop process in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr},
+                )
+            return False
+
+        if self.logger:
+            self.logger.debug(
+                f"Stopped process {pid} in container: {container_id[:12]}", tag="DOCKER"
+            )
+
+        return True
+
+    # Network and Port Methods
+
+    async def wait_for_cdp_ready(self, host_port: int, timeout: int = 10) -> dict:
+        """Wait for the CDP endpoint to be ready.
+
+        Args:
+            host_port: Port to check for CDP endpoint
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            dict: CDP JSON config if ready, None if timeout occurred
+        """
+        import aiohttp
+
+        url = f"http://localhost:{host_port}/json/version"
+
+        for _ in range(timeout):
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(url, timeout=1) as response:
+                        if response.status == 200:
+                            if self.logger:
+                                self.logger.debug(
+                                    f"CDP endpoint ready on port {host_port}",
+                                    tag="DOCKER",
+                                )
+                            cdp_json_config = await response.json()
+                            if self.logger:
+                                self.logger.debug(
+                                    f"CDP JSON config: {cdp_json_config}", tag="DOCKER"
+                                )
+                            return cdp_json_config
+            except Exception:
+                pass
+            await asyncio.sleep(1)
+
+        if self.logger:
+            self.logger.warning(
+                f"CDP endpoint not ready on port {host_port} after {timeout}s timeout",
+                tag="DOCKER",
+            )
+        return None
+
+    def is_port_in_use(self, port: int) -> bool:
+        """Check if a port is already in use on the host.
+
+        Args:
+            port: Port number to check
+
+        Returns:
+            bool: True if port is in use, False otherwise
+        """
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            return s.connect_ex(("localhost", port)) == 0
+
+    def get_next_available_port(self, start_port: int = 9223) -> int:
+        """Get the next available port starting from a given port.
+
+        Args:
+            start_port: Port number to start checking from
+
+        Returns:
+            int: First available port number
+        """
+        port = start_port
+        while self.is_port_in_use(port):
+            port += 1
+        return port
+
+    # Configuration Hash Methods
+
+    def generate_config_hash(self, config_dict: Dict) -> str:
+        """Generate a hash of the configuration for container matching.
+
+        Args:
+            config_dict: Dictionary of configuration parameters
+
+        Returns:
+            str: Hash string uniquely identifying this configuration
+        """
+        # Convert to canonical JSON string and hash
+        config_json = json.dumps(config_dict, sort_keys=True)
+        return hashlib.sha256(config_json.encode()).hexdigest()

crawl4ai/browser/manager copy.py

@@ -0,0 +1,177 @@
+"""Browser manager module for Crawl4AI.
+
+This module provides a central browser management class that uses the
+strategy pattern internally while maintaining the existing API.
+It also implements a page pooling mechanism for improved performance.
+"""
+
+from typing import Optional, Tuple, List
+
+from playwright.async_api import Page, BrowserContext
+
+from ..async_logger import AsyncLogger
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+
+from .strategies import (
+    BaseBrowserStrategy,
+    PlaywrightBrowserStrategy,
+    CDPBrowserStrategy,
+    BuiltinBrowserStrategy,
+    DockerBrowserStrategy
+)
+
+class BrowserManager:
+    """Main interface for browser management in Crawl4AI.
+    
+    This class maintains backward compatibility with the existing implementation
+    while using the strategy pattern internally for different browser types.
+    
+    Attributes:
+        config (BrowserConfig): Configuration object containing all browser settings
+        logger: Logger instance for recording events and errors
+        browser: The browser instance
+        default_context: The default browser context
+        managed_browser: The managed browser instance
+        playwright: The Playwright instance
+        sessions: Dictionary to store session information
+        session_ttl: Session timeout in seconds
+    """
+    
+    def __init__(self, browser_config: Optional[BrowserConfig] = None, logger: Optional[AsyncLogger] = None):
+        """Initialize the BrowserManager with a browser configuration.
+        
+        Args:
+            browser_config: Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+        """
+        self.config = browser_config or BrowserConfig()
+        self.logger = logger
+        
+        # Create strategy based on configuration
+        self.strategy = self._create_strategy()
+        
+        # Initialize state variables for compatibility with existing code
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        
+        # For session management (from existing implementation)
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+    
+    def _create_strategy(self) -> BaseBrowserStrategy:
+        """Create appropriate browser strategy based on configuration.
+        
+        Returns:
+            BaseBrowserStrategy: The selected browser strategy
+        """
+        if self.config.browser_mode == "builtin":
+            return BuiltinBrowserStrategy(self.config, self.logger)
+        elif self.config.browser_mode == "docker":
+            if DockerBrowserStrategy is None:
+                if self.logger:
+                    self.logger.error(
+                        "Docker browser strategy requested but not available. "
+                        "Falling back to PlaywrightBrowserStrategy.",
+                        tag="BROWSER"
+                    )
+                return PlaywrightBrowserStrategy(self.config, self.logger)
+            return DockerBrowserStrategy(self.config, self.logger)
+        elif self.config.browser_mode == "cdp" or self.config.cdp_url or self.config.use_managed_browser:
+            return CDPBrowserStrategy(self.config, self.logger)
+        else:
+            return PlaywrightBrowserStrategy(self.config, self.logger)
+    
+    async def start(self):
+        """Start the browser instance and set up the default context.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Start the strategy
+        await self.strategy.start()
+        
+        # Update legacy references
+        self.browser = self.strategy.browser
+        self.default_context = self.strategy.default_context
+        
+        # Set browser process reference (for CDP strategy)
+        if hasattr(self.strategy, 'browser_process'):
+            self.managed_browser = self.strategy
+        
+        # Set Playwright reference
+        self.playwright = self.strategy.playwright
+        
+        # Sync sessions if needed
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+            self.session_ttl = self.strategy.session_ttl
+        
+        return self
+    
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page for the given configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Delegate to strategy
+        page, context = await self.strategy.get_page(crawlerRunConfig)
+        
+        # Sync sessions if needed
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+        
+        return page, context
+        
+    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
+        """Get multiple pages with the same configuration.
+        
+        This method efficiently creates multiple browser pages using the same configuration,
+        which is useful for parallel crawling of multiple URLs.
+        
+        Args:
+            crawlerRunConfig: Configuration for the pages
+            count: Number of pages to create
+            
+        Returns:
+            List of (Page, Context) tuples
+        """
+        # Delegate to strategy
+        pages = await self.strategy.get_pages(crawlerRunConfig, count)
+        
+        # Sync sessions if needed
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+            
+        return pages
+    
+    # Just for legacy compatibility
+    async def kill_session(self, session_id: str):
+        """Kill a browser session and clean up resources.
+        
+        Args:
+            session_id: The session ID to kill
+        """
+        # Handle kill_session via our strategy if it supports it
+        await self.strategy.kill_session(session_id)
+
+        # sync sessions if needed
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+    
+    async def close(self):
+        """Close the browser and clean up resources."""
+        # Delegate to strategy
+        await self.strategy.close()
+        
+        # Reset legacy references
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        self.sessions = {}

crawl4ai/browser/manager.py

@@ -0,0 +1,853 @@
+"""Browser manager module for Crawl4AI.
+
+This module provides a central browser management class that uses the
+strategy pattern internally while maintaining the existing API.
+It also implements browser pooling for improved performance.
+"""
+
+import asyncio
+import hashlib
+import json
+import math
+from enum import Enum
+from typing import Dict, List, Optional, Tuple, Any
+
+from playwright.async_api import Page, BrowserContext
+
+from ..async_logger import AsyncLogger
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+
+from .strategies import (
+    BaseBrowserStrategy,
+    PlaywrightBrowserStrategy,
+    CDPBrowserStrategy,
+    BuiltinBrowserStrategy,
+    DockerBrowserStrategy
+)
+
+class UnavailableBehavior(Enum):
+    """Behavior when no browser is available."""
+    ON_DEMAND = "on_demand"  # Create new browser on demand
+    PENDING = "pending"      # Wait until a browser is available
+    EXCEPTION = "exception"  # Raise an exception
+
+
+class BrowserManager:
+    """Main interface for browser management and pooling in Crawl4AI.
+    
+    This class maintains backward compatibility with the existing implementation
+    while using the strategy pattern internally for different browser types.
+    It also implements browser pooling for improved performance.
+    
+    Attributes:
+        config (BrowserConfig): Default configuration object for browsers
+        logger (AsyncLogger): Logger instance for recording events and errors
+        browser_pool (Dict): Dictionary to store browser instances by configuration
+        browser_in_use (Dict): Dictionary to track which browsers are in use
+        request_queues (Dict): Queues for pending requests by configuration
+        unavailable_behavior (UnavailableBehavior): Behavior when no browser is available
+    """
+    
+    def __init__(
+        self, 
+        browser_config: Optional[BrowserConfig] = None, 
+        logger: Optional[AsyncLogger] = None,
+        unavailable_behavior: UnavailableBehavior = UnavailableBehavior.EXCEPTION,
+        max_browsers_per_config: int = 10,
+        max_pages_per_browser: int = 5
+        ):
+        """Initialize the BrowserManager with a browser configuration.
+        
+        Args:
+            browser_config: Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+            unavailable_behavior: Behavior when no browser is available
+            max_browsers_per_config: Maximum number of browsers per configuration
+            max_pages_per_browser: Maximum number of pages per browser
+        """
+        self.config = browser_config or BrowserConfig()
+        self.logger = logger
+        self.unavailable_behavior = unavailable_behavior
+        self.max_browsers_per_config = max_browsers_per_config
+        self.max_pages_per_browser = max_pages_per_browser
+        
+        # Browser pool management
+        self.browser_pool = {}            # config_hash -> list of browser strategies
+        self.browser_in_use = {}          # strategy instance -> Boolean
+        self.request_queues = {}          # config_hash -> asyncio.Queue()
+        self._browser_locks = {}          # config_hash -> asyncio.Lock()
+        self._browser_pool_lock = asyncio.Lock()  # Global lock for pool modifications
+        
+        # Page pool management
+        self.page_pool = {}  # (browser_config_hash, crawler_config_hash) -> list of (page, context, strategy)
+        self._page_pool_lock = asyncio.Lock()
+            
+        self.browser_page_counts = {}  # strategy instance -> current page count
+        self._page_count_lock = asyncio.Lock()  # Lock for thread-safe access to page counts
+
+        # For session management (from existing implementation)
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+
+        # For legacy compatibility
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        self.strategy = None
+    
+    def _create_browser_config_hash(self, browser_config: BrowserConfig) -> str:
+        """Create a hash of the browser configuration for browser pooling.
+        
+        Args:
+            browser_config: Browser configuration
+            
+        Returns:
+            str: Hash of the browser configuration
+        """
+        # Convert config to dictionary, excluding any callable objects
+        config_dict = browser_config.__dict__.copy()
+        for key in list(config_dict.keys()):
+            if callable(config_dict[key]):
+                del config_dict[key]
+        
+        # Convert to canonical JSON string
+        config_json = json.dumps(config_dict, sort_keys=True, default=str)
+        
+        # Hash the JSON
+        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
+        return config_hash
+    
+    def _create_strategy(self, browser_config: BrowserConfig) -> BaseBrowserStrategy:
+        """Create appropriate browser strategy based on configuration.
+        
+        Args:
+            browser_config: Browser configuration
+            
+        Returns:
+            BaseBrowserStrategy: The selected browser strategy
+        """
+        if browser_config.browser_mode == "builtin":
+            return BuiltinBrowserStrategy(browser_config, self.logger)
+        elif browser_config.browser_mode == "docker":
+            if DockerBrowserStrategy is None:
+                if self.logger:
+                    self.logger.error(
+                        "Docker browser strategy requested but not available. "
+                        "Falling back to PlaywrightBrowserStrategy.",
+                        tag="BROWSER"
+                    )
+                return PlaywrightBrowserStrategy(browser_config, self.logger)
+            return DockerBrowserStrategy(browser_config, self.logger)
+        elif browser_config.browser_mode == "cdp" or browser_config.cdp_url or browser_config.use_managed_browser:
+            return CDPBrowserStrategy(browser_config, self.logger)
+        else:
+            return PlaywrightBrowserStrategy(browser_config, self.logger)
+    
+    async def initialize_pool(
+        self, 
+        browser_configs: List[BrowserConfig] = None,
+        browsers_per_config: int = 1,
+        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
+    ):
+        """Initialize the browser pool with multiple browser configurations.
+        
+        Args:
+            browser_configs: List of browser configurations to initialize
+            browsers_per_config: Number of browser instances per configuration
+            page_configs: Optional list of (browser_config, crawler_run_config, count) tuples
+                for pre-warming pages
+                
+        Returns:
+            self: For method chaining
+        """
+        if not browser_configs:
+            browser_configs = [self.config]
+        
+        # Calculate how many browsers we'll need based on page_configs
+        browsers_needed = {}
+        if page_configs:
+            for browser_config, _, page_count in page_configs:
+                config_hash = self._create_browser_config_hash(browser_config)
+                # Calculate browsers based on max_pages_per_browser
+                browsers_needed_for_config = math.ceil(page_count / self.max_pages_per_browser)
+                browsers_needed[config_hash] = max(
+                    browsers_needed.get(config_hash, 0),
+                    browsers_needed_for_config
+                )
+        
+        # Adjust browsers_per_config if needed to ensure enough capacity
+        config_browsers_needed = {}
+        for browser_config in browser_configs:
+            config_hash = self._create_browser_config_hash(browser_config)
+            
+            # Estimate browsers needed based on page requirements
+            browsers_for_config = browsers_per_config
+            if config_hash in browsers_needed:
+                browsers_for_config = max(browsers_for_config, browsers_needed[config_hash])
+            
+            config_browsers_needed[config_hash] = browsers_for_config
+            
+            # Update max_browsers_per_config if needed
+            if browsers_for_config > self.max_browsers_per_config:
+                self.max_browsers_per_config = browsers_for_config
+                if self.logger:
+                    self.logger.info(
+                        f"Increased max_browsers_per_config to {browsers_for_config} to accommodate page requirements",
+                        tag="POOL"
+                    )
+        
+        # Initialize locks and queues for each config
+        async with self._browser_pool_lock:
+            for browser_config in browser_configs:
+                config_hash = self._create_browser_config_hash(browser_config)
+                
+                # Initialize lock for this config if needed
+                if config_hash not in self._browser_locks:
+                    self._browser_locks[config_hash] = asyncio.Lock()
+                
+                # Initialize queue for this config if needed
+                if config_hash not in self.request_queues:
+                    self.request_queues[config_hash] = asyncio.Queue()
+                
+                # Initialize pool for this config if needed
+                if config_hash not in self.browser_pool:
+                    self.browser_pool[config_hash] = []
+        
+        # Create browser instances for each configuration in parallel
+        browser_tasks = []
+        
+        for browser_config in browser_configs:
+            config_hash = self._create_browser_config_hash(browser_config)
+            browsers_to_create = config_browsers_needed.get(
+                config_hash, 
+                browsers_per_config
+            ) - len(self.browser_pool.get(config_hash, []))
+            
+            if browsers_to_create <= 0:
+                continue
+                
+            for _ in range(browsers_to_create):
+                # Create a task for each browser initialization
+                task = self._create_and_add_browser(browser_config, config_hash)
+                browser_tasks.append(task)
+        
+        # Wait for all browser initializations to complete
+        if browser_tasks:
+            if self.logger:
+                self.logger.info(f"Initializing {len(browser_tasks)} browsers in parallel...", tag="POOL")
+            await asyncio.gather(*browser_tasks)
+        
+        # Pre-warm pages if requested
+        if page_configs:
+            page_tasks = []
+            for browser_config, crawler_run_config, count in page_configs:
+                task = self._prewarm_pages(browser_config, crawler_run_config, count)
+                page_tasks.append(task)
+            
+            if page_tasks:
+                if self.logger:
+                    self.logger.info(f"Pre-warming pages with {len(page_tasks)} configurations...", tag="POOL")
+                await asyncio.gather(*page_tasks)
+        
+        # Update legacy references
+        if self.browser_pool and next(iter(self.browser_pool.values()), []):
+            strategy = next(iter(self.browser_pool.values()))[0]
+            self.strategy = strategy
+            self.browser = strategy.browser
+            self.default_context = strategy.default_context
+            self.playwright = strategy.playwright
+        
+        return self
+
+    async def _create_and_add_browser(self, browser_config: BrowserConfig, config_hash: str):
+        """Create and add a browser to the pool.
+        
+        Args:
+            browser_config: Browser configuration
+            config_hash: Hash of the configuration
+        """
+        try:
+            strategy = self._create_strategy(browser_config)
+            await strategy.start()
+            
+            async with self._browser_pool_lock:
+                if config_hash not in self.browser_pool:
+                    self.browser_pool[config_hash] = []
+                self.browser_pool[config_hash].append(strategy)
+                self.browser_in_use[strategy] = False
+            
+            if self.logger:
+                self.logger.debug(
+                    f"Added browser to pool: {browser_config.browser_type} "
+                    f"({browser_config.browser_mode})", 
+                    tag="POOL"
+                )
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    f"Failed to create browser: {str(e)}", 
+                    tag="POOL"
+                )
+            raise
+
+    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
+        """Create a signature hash from crawler configuration.
+        
+        Args:
+            crawlerRunConfig: Crawler run configuration
+            
+        Returns:
+            str: Hash of the crawler configuration
+        """
+        config_dict = crawlerRunConfig.__dict__.copy()
+        # Exclude items that do not affect page creation
+        ephemeral_keys = [
+            "session_id",
+            "js_code",
+            "scraping_strategy",
+            "extraction_strategy",
+            "chunking_strategy",
+            "cache_mode",
+            "content_filter",
+            "semaphore_count",
+            "url"
+        ]
+        for key in ephemeral_keys:
+            if key in config_dict:
+                del config_dict[key]
+                
+        # Convert to canonical JSON string
+        config_json = json.dumps(config_dict, sort_keys=True, default=str)
+
+        # Hash the JSON
+        config_hash = hashlib.sha256(config_json.encode("utf-8")).hexdigest()
+        return config_hash            
+
+    async def _prewarm_pages(
+        self, 
+        browser_config: BrowserConfig, 
+        crawler_run_config: CrawlerRunConfig, 
+        count: int
+    ):
+        """Pre-warm pages for a specific configuration.
+        
+        Args:
+            browser_config: Browser configuration
+            crawler_run_config: Crawler run configuration
+            count: Number of pages to pre-warm
+        """
+        try:
+            # Create individual page tasks and run them in parallel
+            browser_config_hash = self._create_browser_config_hash(browser_config)
+            crawler_config_hash = self._make_config_signature(crawler_run_config)            
+            async def get_single_page():
+                strategy = await self.get_available_browser(browser_config)
+                try:
+                    page, context = await strategy.get_page(crawler_run_config)
+                    # Store config hashes on the page object for later retrieval
+                    setattr(page, "_browser_config_hash", browser_config_hash)
+                    setattr(page, "_crawler_config_hash", crawler_config_hash)                    
+                    return page, context, strategy
+                except Exception as e:
+                    # Release the browser back to the pool
+                    await self.release_browser(strategy, browser_config)
+                    raise e
+                
+            # Create tasks for parallel execution
+            page_tasks = [get_single_page() for _ in range(count)]
+            
+            # Execute all page creation tasks in parallel
+            pages_contexts_strategies = await asyncio.gather(*page_tasks)
+            
+            # Add pages to the page pool
+            browser_config_hash = self._create_browser_config_hash(browser_config)
+            crawler_config_hash = self._make_config_signature(crawler_run_config)
+            pool_key = (browser_config_hash, crawler_config_hash)
+            
+            async with self._page_pool_lock:
+                if pool_key not in self.page_pool:
+                    self.page_pool[pool_key] = []
+                
+                # Add all pages to the pool
+                self.page_pool[pool_key].extend(pages_contexts_strategies)
+                
+            if self.logger:
+                self.logger.debug(
+                    f"Pre-warmed {count} pages in parallel with config {crawler_run_config}", 
+                    tag="POOL"
+                )
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    f"Failed to pre-warm pages: {str(e)}", 
+                    tag="POOL"
+                )
+            raise
+    
+    async def get_available_browser(
+        self, 
+        browser_config: Optional[BrowserConfig] = None
+    ) -> BaseBrowserStrategy:
+        """Get an available browser from the pool for the given configuration.
+        
+        Args:
+            browser_config: Browser configuration to match
+            
+        Returns:
+            BaseBrowserStrategy: An available browser strategy
+            
+        Raises:
+            Exception: If no browser is available and behavior is EXCEPTION
+        """
+        browser_config = browser_config or self.config
+        config_hash = self._create_browser_config_hash(browser_config)
+        
+        async with self._browser_locks.get(config_hash, asyncio.Lock()):
+            # Check if we have browsers for this config
+            if config_hash not in self.browser_pool or not self.browser_pool[config_hash]:
+                if self.unavailable_behavior == UnavailableBehavior.ON_DEMAND:
+                    # Create a new browser on demand
+                    if self.logger:
+                        self.logger.info(
+                            f"1> Creating new browser on demand for config {config_hash[:8]}",
+                            tag="POOL"
+                        )
+                    
+                    # Initialize pool for this config if needed
+                    async with self._browser_pool_lock:
+                        if config_hash not in self.browser_pool:
+                            self.browser_pool[config_hash] = []
+                        
+                        strategy = self._create_strategy(browser_config)
+                        await strategy.start()
+
+                        self.browser_pool[config_hash].append(strategy)
+                        self.browser_in_use[strategy] = False
+
+                elif self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
+                    raise Exception(f"No browsers available for configuration {config_hash[:8]}")
+            
+            # Check for an available browser with capacity in the pool
+            for strategy in self.browser_pool[config_hash]:
+                # Check if this browser has capacity for more pages
+                async with self._page_count_lock:
+                    current_pages = self.browser_page_counts.get(strategy, 0)
+
+                    if current_pages < self.max_pages_per_browser:
+                        # Increment the page count
+                        self.browser_page_counts[strategy] = current_pages + 1
+
+                        self.browser_in_use[strategy] = True
+
+                        # Get browser information for better logging
+                        browser_type = getattr(strategy.config, 'browser_type', 'unknown')
+                        browser_mode = getattr(strategy.config, 'browser_mode', 'unknown')
+                        strategy_id = id(strategy)  # Use object ID as a unique identifier
+
+                        if self.logger:
+                            self.logger.debug(
+                                f"Selected browser #{strategy_id} ({browser_type}/{browser_mode}) - " 
+                                f"pages: {current_pages+1}/{self.max_pages_per_browser}",
+                                tag="POOL"
+                            )                        
+
+                        return strategy
+            
+            # All browsers are at capacity or in use
+            if self.unavailable_behavior == UnavailableBehavior.ON_DEMAND:
+                # Check if we've reached the maximum number of browsers
+                if len(self.browser_pool[config_hash]) >= self.max_browsers_per_config:
+                    if self.logger:
+                        self.logger.warning(
+                            f"Maximum browsers reached for config {config_hash[:8]} and all at page capacity",
+                            tag="POOL"
+                        )
+                    if self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
+                        raise Exception("Maximum browsers reached and all at page capacity")
+                
+                # Create a new browser on demand
+                if self.logger:
+                    self.logger.info(
+                        f"2> Creating new browser on demand for config {config_hash[:8]}",
+                        tag="POOL"
+                    )
+                
+                strategy = self._create_strategy(browser_config)
+                await strategy.start()
+                
+                async with self._browser_pool_lock:
+                    self.browser_pool[config_hash].append(strategy)
+                    self.browser_in_use[strategy] = True
+                
+                return strategy
+            
+            # If we get here, either behavior is EXCEPTION or PENDING
+            if self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
+                raise Exception(f"All browsers in use or at page capacity for configuration {config_hash[:8]}")
+            
+            # For PENDING behavior, set up waiting mechanism
+            if config_hash not in self.request_queues:
+                self.request_queues[config_hash] = asyncio.Queue()
+            
+            # Create a future to wait on
+            future = asyncio.Future()
+            await self.request_queues[config_hash].put(future)
+            
+            if self.logger:
+                self.logger.debug(
+                    f"Waiting for available browser for config {config_hash[:8]}",
+                    tag="POOL"
+                )
+            
+            # Wait for a browser to become available
+            strategy = await future
+            return strategy
+
+    async def get_page(
+        self, 
+        crawlerRunConfig: CrawlerRunConfig,
+        browser_config: Optional[BrowserConfig] = None
+    ) -> Tuple[Page, BrowserContext, BaseBrowserStrategy]:
+        """Get a page from the browser pool."""
+        browser_config = browser_config or self.config
+        
+        # Check if we have a pre-warmed page available
+        browser_config_hash = self._create_browser_config_hash(browser_config)
+        crawler_config_hash = self._make_config_signature(crawlerRunConfig)
+        pool_key = (browser_config_hash, crawler_config_hash)
+        
+        # Try to get a page from the pool
+        async with self._page_pool_lock:
+            if pool_key in self.page_pool and self.page_pool[pool_key]:
+                # Get a page from the pool
+                page, context, strategy = self.page_pool[pool_key].pop()
+                
+                # Mark browser as in use (it already is, but ensure consistency)
+                self.browser_in_use[strategy] = True
+                
+                if self.logger:
+                    self.logger.debug(
+                        f"Using pre-warmed page for config {crawler_config_hash[:8]}", 
+                        tag="POOL"
+                    )
+                    
+                # Note: We don't increment page count since it was already counted when created
+                
+                return page, context, strategy
+        
+        # No pre-warmed page available, create a new one
+        # get_available_browser already increments the page count
+        strategy = await self.get_available_browser(browser_config)
+        
+        try:
+            # Get a page from the browser
+            page, context = await strategy.get_page(crawlerRunConfig)
+            
+            # Store config hashes on the page object for later retrieval
+            setattr(page, "_browser_config_hash", browser_config_hash)
+            setattr(page, "_crawler_config_hash", crawler_config_hash)
+            
+            return page, context, strategy
+        except Exception as e:
+            # Release the browser back to the pool and decrement the page count
+            await self.release_browser(strategy, browser_config, decrement_page_count=True)
+            raise e
+        
+    async def release_page(
+        self, 
+        page: Page, 
+        strategy: BaseBrowserStrategy, 
+        browser_config: Optional[BrowserConfig] = None,
+        keep_alive: bool = True,
+        return_to_pool: bool = True
+    ):
+        """Release a page back to the pool."""
+        browser_config = browser_config or self.config
+
+        page_url = page.url if page else None
+        
+        # If not keeping the page alive, close it and decrement count
+        if not keep_alive:
+            try:
+                await page.close()
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(
+                        f"Error closing page: {str(e)}",
+                        tag="POOL"
+                    )
+            # Release the browser with page count decrement
+            await self.release_browser(strategy, browser_config, decrement_page_count=True)
+            return
+        
+        # If returning to pool
+        if return_to_pool:
+            # Get the configuration hashes from the page object
+            browser_config_hash = getattr(page, "_browser_config_hash", None)
+            crawler_config_hash = getattr(page, "_crawler_config_hash", None)
+            
+            if browser_config_hash and crawler_config_hash:
+                pool_key = (browser_config_hash, crawler_config_hash)
+                
+                async with self._page_pool_lock:
+                    if pool_key not in self.page_pool:
+                        self.page_pool[pool_key] = []
+                    
+                    # Add page back to the pool
+                    self.page_pool[pool_key].append((page, page.context, strategy))
+                    
+                    if self.logger:
+                        self.logger.debug(
+                            f"Returned page to pool for config {crawler_config_hash[:8]}, url: {page_url}", 
+                            tag="POOL"
+                        )
+                    
+                    # Note: We don't decrement the page count here since the page is still "in use" 
+                    # from the browser's perspective, just in our pool
+                    return
+            else:
+                # If we can't identify the configuration, log a warning
+                if self.logger:
+                    self.logger.warning(
+                        "Cannot return page to pool - missing configuration hashes", 
+                        tag="POOL"
+                    )
+        
+        # If we got here, we couldn't return to pool, so just release the browser
+        await self.release_browser(strategy, browser_config, decrement_page_count=True)
+    
+    async def release_browser(
+        self, 
+        strategy: BaseBrowserStrategy, 
+        browser_config: Optional[BrowserConfig] = None,
+        decrement_page_count: bool = True
+    ):
+        """Release a browser back to the pool."""
+        browser_config = browser_config or self.config
+        config_hash = self._create_browser_config_hash(browser_config)
+        
+        # Decrement page count
+        if decrement_page_count:
+            async with self._page_count_lock:
+                current_count = self.browser_page_counts.get(strategy, 1)
+                self.browser_page_counts[strategy] = max(0, current_count - 1)
+                
+                if self.logger:
+                    self.logger.debug(
+                        f"Decremented page count for browser (now: {self.browser_page_counts[strategy]})",
+                        tag="POOL"
+                    )
+        
+        # Mark as not in use
+        self.browser_in_use[strategy] = False
+        
+        # Process any waiting requests
+        if config_hash in self.request_queues and not self.request_queues[config_hash].empty():
+            future = await self.request_queues[config_hash].get()
+            if not future.done():
+                future.set_result(strategy)
+
+    async def get_pages(
+        self, 
+        crawlerRunConfig: CrawlerRunConfig, 
+        count: int = 1,
+        browser_config: Optional[BrowserConfig] = None
+    ) -> List[Tuple[Page, BrowserContext, BaseBrowserStrategy]]:
+        """Get multiple pages from the browser pool.
+        
+        Args:
+            crawlerRunConfig: Configuration for the crawler run
+            count: Number of pages to get
+            browser_config: Browser configuration to use
+            
+        Returns:
+            List of (Page, Context, Strategy) tuples
+        """
+        results = []
+        for _ in range(count):
+            try:
+                result = await self.get_page(crawlerRunConfig, browser_config)
+                results.append(result)
+            except Exception as e:
+                # Release any pages we've already gotten
+                for page, _, strategy in results:
+                    await self.release_page(page, strategy, browser_config)
+                raise e
+        
+        return results
+
+    async def get_page_pool_status(self) -> Dict[str, Any]:
+        """Get information about the page pool status.
+        
+        Returns:
+            Dict with page pool status information
+        """
+        status = {
+            "total_pooled_pages": 0,
+            "configs": {}
+        }
+        
+        async with self._page_pool_lock:
+            for (browser_hash, crawler_hash), pages in self.page_pool.items():
+                config_key = f"{browser_hash[:8]}_{crawler_hash[:8]}"
+                status["configs"][config_key] = len(pages)
+                status["total_pooled_pages"] += len(pages)
+        
+        if self.logger:
+            self.logger.debug(
+                f"Page pool status: {status['total_pooled_pages']} pages available",
+                tag="POOL"
+            )
+        
+        return status
+
+    async def get_pool_status(self) -> Dict[str, Any]:
+        """Get information about the browser pool status.
+        
+        Returns:
+            Dict with pool status information
+        """
+        status = {
+            "total_browsers": 0,
+            "browsers_in_use": 0,
+            "total_pages": 0,
+            "configs": {}
+        }
+        
+        for config_hash, strategies in self.browser_pool.items():
+            config_pages = 0
+            in_use = 0
+            
+            for strategy in strategies:
+                is_in_use = self.browser_in_use.get(strategy, False)
+                if is_in_use:
+                    in_use += 1
+                
+                # Get page count for this browser
+                try:
+                    page_count = len(await strategy.get_opened_pages())
+                    config_pages += page_count
+                except Exception as e:
+                    if self.logger:
+                        self.logger.error(f"Error getting page count: {str(e)}", tag="POOL")
+            
+            config_status = {
+                "total_browsers": len(strategies),
+                "browsers_in_use": in_use,
+                "pages_open": config_pages,
+                "waiting_requests": self.request_queues.get(config_hash, asyncio.Queue()).qsize(),
+                "max_capacity": len(strategies) * self.max_pages_per_browser,
+                "utilization_pct": round((config_pages / (len(strategies) * self.max_pages_per_browser)) * 100, 1) 
+                    if strategies else 0
+            }
+            
+            status["configs"][config_hash] = config_status
+            status["total_browsers"] += config_status["total_browsers"]
+            status["browsers_in_use"] += config_status["browsers_in_use"]
+            status["total_pages"] += config_pages
+        
+        # Add overall utilization
+        if status["total_browsers"] > 0:
+            max_capacity = status["total_browsers"] * self.max_pages_per_browser
+            status["overall_utilization_pct"] = round((status["total_pages"] / max_capacity) * 100, 1)
+        else:
+            status["overall_utilization_pct"] = 0
+        
+        return status
+
+    async def start(self):
+        """Start at least one browser instance in the pool.
+        
+        This method is kept for backward compatibility.
+        
+        Returns:
+            self: For method chaining
+        """
+        await self.initialize_pool([self.config], 1)
+        return self
+        
+    async def kill_session(self, session_id: str):
+        """Kill a browser session and clean up resources.
+        
+        Delegated to the strategy. This method is kept for backward compatibility.
+        
+        Args:
+            session_id: The session ID to kill
+        """
+        if not self.strategy:
+            return
+            
+        await self.strategy.kill_session(session_id)
+        
+        # Sync sessions
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+    
+    async def close(self):
+        """Close all browsers in the pool and clean up resources."""
+        # Close all browsers in the pool
+        for strategies in self.browser_pool.values():
+            for strategy in strategies:
+                try:
+                    await strategy.close()
+                except Exception as e:
+                    if self.logger:
+                        self.logger.error(
+                            f"Error closing browser: {str(e)}",
+                            tag="POOL"
+                        )
+        
+        # Clear pool data
+        self.browser_pool = {}
+        self.browser_in_use = {}
+        
+        # Reset legacy references
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        self.strategy = None
+        self.sessions = {}
+
+
+async def create_browser_manager(
+    browser_config: Optional[BrowserConfig] = None,
+    logger: Optional[AsyncLogger] = None,
+    unavailable_behavior: UnavailableBehavior = UnavailableBehavior.EXCEPTION,
+    max_browsers_per_config: int = 10,
+    initial_pool_size: int = 1,
+    page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
+) -> BrowserManager:
+    """Factory function to create and initialize a BrowserManager.
+    
+    Args:
+        browser_config: Configuration for the browsers
+        logger: Logger for recording events
+        unavailable_behavior: Behavior when no browser is available
+        max_browsers_per_config: Maximum browsers per configuration
+        initial_pool_size: Initial number of browsers per configuration
+        page_configs: Optional configurations for pre-warming pages
+        
+    Returns:
+        Initialized BrowserManager
+    """
+    manager = BrowserManager(
+        browser_config=browser_config,
+        logger=logger,
+        unavailable_behavior=unavailable_behavior,
+        max_browsers_per_config=max_browsers_per_config
+    )
+    
+    await manager.initialize_pool(
+        [browser_config] if browser_config else None,
+        initial_pool_size,
+        page_configs
+    )
+    
+    return manager
+
+
+
+
+

crawl4ai/browser/models.py

@@ -0,0 +1,143 @@
+"""Docker configuration module for Crawl4AI browser automation.
+
+This module provides configuration classes for Docker-based browser automation,
+allowing flexible configuration of Docker containers for browsing.
+"""
+
+from typing import Dict, List, Optional
+
+
+class DockerConfig:
+    """Configuration for Docker-based browser automation.
+    
+    This class contains Docker-specific settings to avoid cluttering BrowserConfig.
+    
+    Attributes:
+        mode (str): Docker operation mode - "connect" or "launch".
+            - "connect": Uses a container with Chrome already running
+            - "launch": Dynamically configures and starts Chrome in container
+        image (str): Docker image to use. If None, defaults from DockerUtils are used.
+        registry_file (str): Path to container registry file for persistence.
+        persistent (bool): Keep container running after browser closes.
+        remove_on_exit (bool): Remove container on exit when not persistent.
+        network (str): Docker network to use.
+        volumes (List[str]): Volume mappings (e.g., ["host_path:container_path"]).
+        env_vars (Dict[str, str]): Environment variables to set in container.
+        extra_args (List[str]): Additional docker run arguments.
+        host_port (int): Host port to map to container's 9223 port.
+        user_data_dir (str): Path to user data directory on host.
+        container_user_data_dir (str): Path to user data directory in container.
+    """
+    
+    def __init__(
+        self,
+        mode: str = "connect",                     # "connect" or "launch" 
+        image: Optional[str] = None,               # Docker image to use
+        registry_file: Optional[str] = None,       # Path to registry file
+        persistent: bool = False,                  # Keep container running after browser closes
+        remove_on_exit: bool = True,               # Remove container on exit when not persistent
+        network: Optional[str] = None,             # Docker network to use
+        volumes: List[str] = None,                 # Volume mappings
+        cpu_limit: float = 1.0,                    # CPU limit for the container
+        memory_limit: str = "1.5g",                # Memory limit for the container
+        env_vars: Dict[str, str] = None,           # Environment variables
+        host_port: Optional[int] = None,           # Host port to map to container's 9223
+        user_data_dir: Optional[str] = None,       # Path to user data directory on host
+        container_user_data_dir: str = "/data",    # Path to user data directory in container
+        extra_args: List[str] = None,              # Additional docker run arguments
+    ):
+        """Initialize Docker configuration.
+        
+        Args:
+            mode: Docker operation mode ("connect" or "launch")
+            image: Docker image to use
+            registry_file: Path to container registry file
+            persistent: Whether to keep container running after browser closes
+            remove_on_exit: Whether to remove container on exit when not persistent
+            network: Docker network to use
+            volumes: Volume mappings as list of strings
+            cpu_limit: CPU limit for the container
+            memory_limit: Memory limit for the container
+            env_vars: Environment variables as dictionary
+            extra_args: Additional docker run arguments
+            host_port: Host port to map to container's 9223
+            user_data_dir: Path to user data directory on host
+            container_user_data_dir: Path to user data directory in container
+        """
+        self.mode = mode
+        self.image = image  # If None, defaults will be used from DockerUtils
+        self.registry_file = registry_file
+        self.persistent = persistent
+        self.remove_on_exit = remove_on_exit
+        self.network = network
+        self.volumes = volumes or []
+        self.cpu_limit = cpu_limit
+        self.memory_limit = memory_limit
+        self.env_vars = env_vars or {}
+        self.extra_args = extra_args or []
+        self.host_port = host_port
+        self.user_data_dir = user_data_dir
+        self.container_user_data_dir = container_user_data_dir
+    
+    def to_dict(self) -> Dict:
+        """Convert this configuration to a dictionary.
+        
+        Returns:
+            Dictionary representation of this configuration
+        """
+        return {
+            "mode": self.mode,
+            "image": self.image,
+            "registry_file": self.registry_file,
+            "persistent": self.persistent,
+            "remove_on_exit": self.remove_on_exit,
+            "network": self.network,
+            "volumes": self.volumes,
+            "cpu_limit": self.cpu_limit,
+            "memory_limit": self.memory_limit,
+            "env_vars": self.env_vars,
+            "extra_args": self.extra_args,
+            "host_port": self.host_port,
+            "user_data_dir": self.user_data_dir,
+            "container_user_data_dir": self.container_user_data_dir
+        }
+        
+    @staticmethod
+    def from_kwargs(kwargs: Dict) -> "DockerConfig":
+        """Create a DockerConfig from a dictionary of keyword arguments.
+        
+        Args:
+            kwargs: Dictionary of configuration options
+            
+        Returns:
+            New DockerConfig instance
+        """
+        return DockerConfig(
+            mode=kwargs.get("mode", "connect"),
+            image=kwargs.get("image"),
+            registry_file=kwargs.get("registry_file"),
+            persistent=kwargs.get("persistent", False),
+            remove_on_exit=kwargs.get("remove_on_exit", True),
+            network=kwargs.get("network"),
+            volumes=kwargs.get("volumes"),
+            cpu_limit=kwargs.get("cpu_limit", 1.0),
+            memory_limit=kwargs.get("memory_limit", "1.5g"),
+            env_vars=kwargs.get("env_vars"),
+            extra_args=kwargs.get("extra_args"),
+            host_port=kwargs.get("host_port"),
+            user_data_dir=kwargs.get("user_data_dir"),
+            container_user_data_dir=kwargs.get("container_user_data_dir", "/data")
+        )
+        
+    def clone(self, **kwargs) -> "DockerConfig":
+        """Create a copy of this configuration with updated values.
+        
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+            
+        Returns:
+            DockerConfig: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return DockerConfig.from_kwargs(config_dict)

crawl4ai/browser/profiles.py

@@ -0,0 +1,457 @@
+"""Browser profile management module for Crawl4AI.
+
+This module provides functionality for creating and managing browser profiles
+that can be used for authenticated browsing.
+"""
+
+import os
+import asyncio
+import signal
+import sys
+import datetime
+import uuid
+import shutil
+from typing import List, Dict, Optional, Any
+from colorama import Fore, Style, init
+
+from ..async_configs import BrowserConfig
+from ..async_logger import AsyncLogger, AsyncLoggerBase
+from ..utils import get_home_folder
+
+class BrowserProfileManager:
+    """Manages browser profiles for Crawl4AI.
+    
+    This class provides functionality to create and manage browser profiles
+    that can be used for authenticated browsing with Crawl4AI.
+    
+    Profiles are stored by default in ~/.crawl4ai/profiles/
+    """
+    
+    def __init__(self, logger: Optional[AsyncLoggerBase] = None):
+        """Initialize the BrowserProfileManager.
+        
+        Args:
+            logger: Logger for outputting messages. If None, a default AsyncLogger is created.
+        """
+        # Initialize colorama for colorful terminal output
+        init()
+        
+        # Create a logger if not provided
+        if logger is None:
+            self.logger = AsyncLogger(verbose=True)
+        elif not isinstance(logger, AsyncLoggerBase):
+            self.logger = AsyncLogger(verbose=True)
+        else:
+            self.logger = logger
+            
+        # Ensure profiles directory exists
+        self.profiles_dir = os.path.join(get_home_folder(), "profiles")
+        os.makedirs(self.profiles_dir, exist_ok=True)
+    
+    async def create_profile(self, 
+                          profile_name: Optional[str] = None, 
+                          browser_config: Optional[BrowserConfig] = None) -> Optional[str]:
+        """Create a browser profile interactively.
+        
+        Args:
+            profile_name: Name for the profile. If None, a name is generated.
+            browser_config: Configuration for the browser. If None, a default configuration is used.
+                
+        Returns:
+            Path to the created profile directory, or None if creation failed
+        """
+        # Create default browser config if none provided
+        if browser_config is None:
+            browser_config = BrowserConfig(
+                browser_type="chromium",
+                headless=False,  # Must be visible for user interaction
+                verbose=True
+            )
+        else:
+            # Ensure headless is False for user interaction
+            browser_config.headless = False
+            
+        # Generate profile name if not provided
+        if not profile_name:
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"profile_{timestamp}_{uuid.uuid4().hex[:6]}"
+            
+        # Sanitize profile name (replace spaces and special chars)
+        profile_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in profile_name)
+        
+        # Set user data directory
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        os.makedirs(profile_path, exist_ok=True)
+        
+        # Print instructions for the user with colorama formatting
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="PROFILE")
+        self.logger.info(f"Creating browser profile: {Fore.GREEN}{profile_name}{Style.RESET_ALL}", tag="PROFILE")
+        self.logger.info(f"Profile directory: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+        
+        self.logger.info("\nInstructions:", tag="PROFILE")
+        self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE")
+        self.logger.info(f"2. {Fore.CYAN}Log in to websites{Style.RESET_ALL}, configure settings, etc. as needed.", tag="PROFILE")
+        self.logger.info(f"3. When you're done, {Fore.YELLOW}press 'q' in this terminal{Style.RESET_ALL} to close the browser.", tag="PROFILE")
+        self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE")
+        self.logger.info(f"{border}\n", tag="PROFILE")
+        
+        # Import the necessary classes with local imports to avoid circular references
+        from .strategies import CDPBrowserStrategy
+        
+        # Set browser config to use the profile path
+        browser_config.user_data_dir = profile_path
+        
+        # Create a CDP browser strategy for the profile creation
+        browser_strategy = CDPBrowserStrategy(browser_config, self.logger)
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="PROFILE")
+            await browser_strategy.close()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Profile creation interrupted. Profile may be incomplete.", tag="PROFILE")
+                sys.exit(1)
+                
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user is done with the browser
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' when you've finished using the browser...{Style.RESET_ALL}", tag="PROFILE")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser and saving profile...{Style.RESET_ALL}", tag="PROFILE")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if browser_strategy.browser_process and browser_strategy.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="PROFILE")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+        
+        try:
+            # Start the browser
+            await browser_strategy.start()
+            
+            # Check if browser started successfully
+            if not browser_strategy.browser_process:
+                self.logger.error("Failed to start browser process.", tag="PROFILE")
+                return None
+            
+            self.logger.info(f"Browser launched. {Fore.CYAN}Waiting for you to finish...{Style.RESET_ALL}", tag="PROFILE") 
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for either the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_strategy.browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_strategy.browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="PROFILE")
+                await browser_strategy.close()
+            
+            self.logger.success(f"Browser closed. Profile saved at: {Fore.GREEN}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+                
+        except Exception as e:
+            self.logger.error(f"Error creating profile: {str(e)}", tag="PROFILE")
+            await browser_strategy.close()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await browser_strategy.close()
+        
+        # Return the profile path
+        return profile_path
+    
+    def list_profiles(self) -> List[Dict[str, Any]]:
+        """List all available browser profiles.
+        
+        Returns:
+            List of dictionaries containing profile information
+        """
+        if not os.path.exists(self.profiles_dir):
+            return []
+            
+        profiles = []
+        
+        for name in os.listdir(self.profiles_dir):
+            profile_path = os.path.join(self.profiles_dir, name)
+            
+            # Skip if not a directory
+            if not os.path.isdir(profile_path):
+                continue
+                
+            # Check if this looks like a valid browser profile
+            # For Chromium: Look for Preferences file
+            # For Firefox: Look for prefs.js file
+            is_valid = False
+            
+            if os.path.exists(os.path.join(profile_path, "Preferences")) or \
+               os.path.exists(os.path.join(profile_path, "Default", "Preferences")):
+                is_valid = "chromium"
+            elif os.path.exists(os.path.join(profile_path, "prefs.js")):
+                is_valid = "firefox"
+                
+            if is_valid:
+                # Get creation time
+                created = datetime.datetime.fromtimestamp(
+                    os.path.getctime(profile_path)
+                )
+                
+                profiles.append({
+                    "name": name,
+                    "path": profile_path,
+                    "created": created,
+                    "type": is_valid
+                })
+                
+        # Sort by creation time, newest first
+        profiles.sort(key=lambda x: x["created"], reverse=True)
+        
+        return profiles
+    
+    def get_profile_path(self, profile_name: str) -> Optional[str]:
+        """Get the full path to a profile by name.
+        
+        Args:
+            profile_name: Name of the profile (not the full path)
+            
+        Returns:
+            Full path to the profile directory, or None if not found
+        """
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            # Check if profile_name itself is full path
+            if os.path.isabs(profile_name):
+                profile_path = profile_name
+            else:
+                return None
+        
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return None  # Not a valid browser profile
+            
+        return profile_path
+    
+    def delete_profile(self, profile_name_or_path: str) -> bool:
+        """Delete a browser profile by name or path.
+        
+        Args:
+            profile_name_or_path: Name of the profile or full path to profile directory
+            
+        Returns:
+            True if the profile was deleted successfully, False otherwise
+        """
+        # Determine if input is a name or a path
+        if os.path.isabs(profile_name_or_path):
+            # Full path provided
+            profile_path = profile_name_or_path
+        else:
+            # Just a name provided, construct path
+            profile_path = os.path.join(self.profiles_dir, profile_name_or_path)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            return False
+            
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return False  # Not a valid browser profile
+            
+        # Delete the profile directory
+        try:
+            shutil.rmtree(profile_path)
+            return True
+        except Exception:
+            return False
+            
+    async def interactive_manager(self, crawl_callback=None):
+        """Launch an interactive profile management console.
+        
+        Args:
+            crawl_callback: Function to call when selecting option to use 
+                a profile for crawling. It will be called with (profile_path, url).
+        """
+        while True:
+            self.logger.info(f"\n{Fore.CYAN}Profile Management Options:{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"1. {Fore.GREEN}Create a new profile{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"2. {Fore.YELLOW}List available profiles{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"3. {Fore.RED}Delete a profile{Style.RESET_ALL}", tag="MENU")
+            
+            # Only show crawl option if callback provided
+            if crawl_callback:
+                self.logger.info(f"4. {Fore.CYAN}Use a profile to crawl a website{Style.RESET_ALL}", tag="MENU")
+                self.logger.info(f"5. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "5"
+            else:
+                self.logger.info(f"4. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "4"
+            
+            choice = input(f"\n{Fore.CYAN}Enter your choice (1-{exit_option}): {Style.RESET_ALL}")
+            
+            if choice == "1":
+                # Create new profile
+                name = input(f"{Fore.GREEN}Enter a name for the new profile (or press Enter for auto-generated name): {Style.RESET_ALL}")
+                await self.create_profile(name or None)
+                
+            elif choice == "2":
+                # List profiles
+                profiles = self.list_profiles()
+                
+                if not profiles:
+                    self.logger.warning("  No profiles found. Create one first with option 1.", tag="PROFILES")
+                    continue
+                
+                # Print profile information with colorama formatting
+                self.logger.info("\nAvailable profiles:", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {Fore.CYAN}{profile['name']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Path: {Fore.YELLOW}{profile['path']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES")
+                    self.logger.info(f"    Browser type: {profile['type']}", tag="PROFILES")
+                    self.logger.info("", tag="PROFILES")  # Empty line for spacing
+                
+            elif choice == "3":
+                # Delete profile
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found to delete", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to delete
+                profile_idx = input(f"{Fore.RED}Enter the number of the profile to delete (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_name = profiles[idx]["name"]
+                        self.logger.info(f"Deleting profile: {Fore.YELLOW}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                        
+                        # Confirm deletion
+                        confirm = input(f"{Fore.RED}Are you sure you want to delete this profile? (y/n): {Style.RESET_ALL}")
+                        if confirm.lower() == 'y':
+                            success = self.delete_profile(profiles[idx]["path"])
+                            
+                            if success:
+                                self.logger.success(f"Profile {Fore.GREEN}{profile_name}{Style.RESET_ALL} deleted successfully", tag="PROFILES")
+                            else:
+                                self.logger.error(f"Failed to delete profile {Fore.RED}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == "4" and crawl_callback:
+                # Use profile to crawl a site
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found. Create one first.", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to use
+                profile_idx = input(f"{Fore.CYAN}Enter the number of the profile to use (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_path = profiles[idx]["path"]
+                        url = input(f"{Fore.CYAN}Enter the URL to crawl: {Style.RESET_ALL}")
+                        if url:
+                            # Call the provided crawl callback
+                            await crawl_callback(profile_path, url)
+                        else:
+                            self.logger.error("No URL provided", tag="CRAWL")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif (choice == "4" and not crawl_callback) or (choice == "5" and crawl_callback):
+                # Exit
+                self.logger.info("Exiting profile management", tag="MENU")
+                break
+                
+            else:
+                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")

crawl4ai/browser/strategies/__init__.py

@@ -0,0 +1,13 @@
+from .base import BaseBrowserStrategy
+from .cdp import CDPBrowserStrategy
+from .docker_strategy import DockerBrowserStrategy
+from .playwright import PlaywrightBrowserStrategy
+from .builtin import BuiltinBrowserStrategy
+
+__all__ = [
+    "BrowserStrategy",
+    "CDPBrowserStrategy",
+    "DockerBrowserStrategy",
+    "PlaywrightBrowserStrategy",
+    "BuiltinBrowserStrategy",
+]

crawl4ai/browser/strategies/base.py

@@ -0,0 +1,601 @@
+"""Browser strategies module for Crawl4AI.
+
+This module implements the browser strategy pattern for different
+browser implementations, including Playwright, CDP, and builtin browsers.
+"""
+
+from abc import ABC, abstractmethod
+import asyncio
+import json
+import hashlib
+import os
+import time
+from typing import Optional, Tuple, List
+
+from playwright.async_api import BrowserContext, Page
+
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig, CrawlerRunConfig
+from ...config import DOWNLOAD_PAGE_TIMEOUT
+from ...js_snippet import load_js_script
+from ..utils import get_playwright
+
+
+class BaseBrowserStrategy(ABC):
+    """Base class for all browser strategies.
+    
+    This abstract class defines the interface that all browser strategies
+    must implement. It handles common functionality like context caching,
+    browser configuration, and session management.
+    """
+    
+    _playwright_instance = None
+    
+    @classmethod
+    async def get_playwright(cls):
+        """Get or create a shared Playwright instance.
+        
+        Returns:
+            Playwright: The shared Playwright instance
+        """
+        # For now I dont want Singleton pattern for Playwright
+        if cls._playwright_instance is None or True:
+            cls._playwright_instance = await get_playwright()
+        return cls._playwright_instance
+        
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the strategy with configuration and logger.
+        
+        Args:
+            config: Browser configuration
+            logger: Logger for recording events and errors
+        """
+        self.config = config
+        self.logger = logger
+        self.browser = None
+        self.default_context = None
+        
+        # Context management
+        self.contexts_by_config = {}  # config_signature -> context
+
+        self._contexts_lock = asyncio.Lock()
+        
+        # Session management
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes default
+        
+        # Playwright instance
+        self.playwright = None
+    
+    @abstractmethod
+    async def start(self):
+        """Start the browser.
+        
+        This method should be implemented by concrete strategies to initialize 
+        the browser in the appropriate way (direct launch, CDP connection, etc.)
+        
+        Returns:
+            self: For method chaining
+        """
+        # Base implementation gets the playwright instance
+        self.playwright = await self.get_playwright()
+        return self
+    
+    @abstractmethod
+    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        pass
+
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page with specified configuration.
+        
+        This method should be implemented by concrete strategies to create
+        or retrieve a page according to their browser management approach.
+        
+        Args:
+            crawlerRunConfig: Crawler run configuration
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Clean up expired sessions first
+        self._cleanup_expired_sessions()
+
+        # If a session_id is provided and we already have it, reuse that page + context
+        if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions:
+            context, page, _ = self.sessions[crawlerRunConfig.session_id]
+            # Update last-used timestamp
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            return page, context
+        
+        page, context = await self._generate_page(crawlerRunConfig)
+
+        import uuid
+        setattr(page, "guid", uuid.uuid4())
+
+        # If a session_id is specified, store this session so we can reuse later
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            
+        return page, context        
+        pass
+    
+    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
+        """Get multiple pages with the same configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration for the pages
+            count: Number of pages to create
+            
+        Returns:
+            List of (Page, Context) tuples
+        """
+        pages = []
+        for _ in range(count):
+            page, context = await self.get_page(crawlerRunConfig)
+            pages.append((page, context))
+        return pages
+       
+    async def get_opened_pages(self) -> List[Page]:
+        """Get all opened pages in the
+        browser.
+        """
+        return [page for context in self.contexts_by_config.values() for page in context.pages]
+
+    def _build_browser_args(self) -> dict:
+        """Build browser launch arguments from config.
+        
+        Returns:
+            dict: Browser launch arguments for Playwright
+        """
+        # Define common browser arguments that improve performance and stability
+        args = [
+            "--no-sandbox",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--window-position=0,0",
+            "--ignore-certificate-errors",
+            "--ignore-certificate-errors-spki-list",
+            "--window-position=400,0",
+            "--force-color-profile=srgb",
+            "--mute-audio",
+            "--disable-gpu",
+            "--disable-gpu-compositing",
+            "--disable-software-rasterizer",
+            "--disable-dev-shm-usage",
+            "--disable-infobars",
+            "--disable-blink-features=AutomationControlled",
+            "--disable-renderer-backgrounding",
+            "--disable-ipc-flooding-protection",
+            "--disable-background-timer-throttling",
+            f"--window-size={self.config.viewport_width},{self.config.viewport_height}",
+        ]
+
+        # Define browser disable options for light mode
+        browser_disable_options = [
+            "--disable-backgrounding-occluded-windows",
+            "--disable-breakpad",
+            "--disable-client-side-phishing-detection",
+            "--disable-component-extensions-with-background-pages",
+            "--disable-default-apps",
+            "--disable-extensions",
+            "--disable-features=TranslateUI",
+            "--disable-hang-monitor",
+            "--disable-popup-blocking",
+            "--disable-prompt-on-repost",
+            "--disable-sync",
+            "--metrics-recording-only",
+            "--password-store=basic",
+            "--use-mock-keychain",
+        ]
+
+        # Apply light mode settings if enabled
+        if self.config.light_mode:
+            args.extend(browser_disable_options)
+
+        # Apply text mode settings if enabled (disables images, JS, etc)
+        if self.config.text_mode:
+            args.extend([
+                "--blink-settings=imagesEnabled=false",
+                "--disable-remote-fonts",
+                "--disable-images",
+                "--disable-javascript",
+                "--disable-software-rasterizer",
+                "--disable-dev-shm-usage",
+            ])
+
+        # Add any extra arguments from the config
+        if self.config.extra_args:
+            args.extend(self.config.extra_args)
+
+        # Build the core browser args dictionary
+        browser_args = {"headless": self.config.headless, "args": args}
+
+        # Add chrome channel if specified
+        if self.config.chrome_channel:
+            browser_args["channel"] = self.config.chrome_channel
+
+        # Configure downloads
+        if self.config.accept_downloads:
+            browser_args["downloads_path"] = self.config.downloads_path or os.path.join(
+                os.getcwd(), "downloads"
+            )
+            os.makedirs(browser_args["downloads_path"], exist_ok=True)
+
+        # Check for user data directory
+        if self.config.user_data_dir:
+            # Ensure the directory exists
+            os.makedirs(self.config.user_data_dir, exist_ok=True)
+            browser_args["user_data_dir"] = self.config.user_data_dir
+        
+        # Configure proxy settings
+        if self.config.proxy or self.config.proxy_config:
+            from playwright.async_api import ProxySettings
+
+            proxy_settings = (
+                ProxySettings(server=self.config.proxy)
+                if self.config.proxy
+                else ProxySettings(
+                    server=self.config.proxy_config.server,
+                    username=self.config.proxy_config.username,
+                    password=self.config.proxy_config.password,
+                )
+            )
+            browser_args["proxy"] = proxy_settings
+
+        return browser_args
+    
+    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
+        """Create a signature hash from configuration for context caching.
+        
+        Converts the crawlerRunConfig into a dict, excludes ephemeral fields,
+        then returns a hash of the sorted JSON. This yields a stable signature
+        that identifies configurations requiring a unique browser context.
+        
+        Args:
+            crawlerRunConfig: Crawler run configuration
+            
+        Returns:
+            str: Unique hash for this configuration
+        """
+        config_dict = crawlerRunConfig.__dict__.copy()
+        # Exclude items that do not affect browser-level setup
+        ephemeral_keys = [
+            "session_id",
+            "js_code",
+            "scraping_strategy",
+            "extraction_strategy",
+            "chunking_strategy",
+            "cache_mode",
+            "content_filter",
+            "semaphore_count",
+            "url"
+        ]
+        for key in ephemeral_keys:
+            if key in config_dict:
+                del config_dict[key]
+                
+        # Convert to canonical JSON string
+        signature_json = json.dumps(config_dict, sort_keys=True, default=str)
+
+        # Hash the JSON so we get a compact, unique string
+        signature_hash = hashlib.sha256(signature_json.encode("utf-8")).hexdigest()
+        return signature_hash
+        
+    async def create_browser_context(self, crawlerRunConfig: Optional[CrawlerRunConfig] = None) -> BrowserContext:
+        """Creates and returns a new browser context with configured settings.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            BrowserContext: Browser context object with the specified configurations
+        """
+        if not self.browser:
+            raise ValueError("Browser must be initialized before creating context")
+            
+        # Base settings
+        user_agent = self.config.headers.get("User-Agent", self.config.user_agent) 
+        viewport_settings = {
+            "width": self.config.viewport_width,
+            "height": self.config.viewport_height,
+        }
+        proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
+        
+        # Define blocked extensions for resource optimization
+        blocked_extensions = [
+            # Images
+            "jpg", "jpeg", "png", "gif", "webp", "svg", "ico", "bmp", "tiff", "psd",
+            # Fonts
+            "woff", "woff2", "ttf", "otf", "eot",
+            # Media
+            "mp4", "webm", "ogg", "avi", "mov", "wmv", "flv", "m4v", "mp3", "wav", "aac",
+            "m4a", "opus", "flac",
+            # Documents
+            "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx",
+            # Archives
+            "zip", "rar", "7z", "tar", "gz",
+            # Scripts and data
+            "xml", "swf", "wasm",
+        ]
+
+        # Common context settings
+        context_settings = {
+            "user_agent": user_agent,
+            "viewport": viewport_settings,
+            "proxy": proxy_settings,
+            "accept_downloads": self.config.accept_downloads,
+            "storage_state": self.config.storage_state,
+            "ignore_https_errors": self.config.ignore_https_errors,
+            "device_scale_factor": 1.0,
+            "java_script_enabled": self.config.java_script_enabled,
+        }
+        
+        # Apply text mode settings if enabled
+        if self.config.text_mode:
+            text_mode_settings = {
+                "has_touch": False,
+                "is_mobile": False,
+                "java_script_enabled": False,  # Disable javascript in text mode
+            }
+            # Update context settings with text mode settings
+            context_settings.update(text_mode_settings)
+            if self.logger:
+                self.logger.debug("Text mode enabled for browser context", tag="BROWSER")
+        
+        # Handle storage state properly - this is key for persistence
+        if self.config.storage_state:
+            if self.logger:
+                if isinstance(self.config.storage_state, str):
+                    self.logger.debug(f"Using storage state from file: {self.config.storage_state}", tag="BROWSER")
+                else:
+                    self.logger.debug("Using storage state from config object", tag="BROWSER")
+
+        if self.config.user_data_dir:
+            # For CDP-based browsers, storage persistence is typically handled by the user_data_dir
+            # at the browser level, but we'll create a storage_state location for Playwright as well
+            storage_path = os.path.join(self.config.user_data_dir, "storage_state.json")
+            if not os.path.exists(storage_path):
+                # Create parent directory if it doesn't exist
+                os.makedirs(os.path.dirname(storage_path), exist_ok=True)
+                with open(storage_path, "w") as f:
+                    json.dump({}, f)
+            self.config.storage_state = storage_path
+
+            if self.logger:
+                self.logger.debug(f"Using user data directory: {self.config.user_data_dir}", tag="BROWSER")
+        
+        # Apply crawler-specific configurations if provided
+        if crawlerRunConfig:
+            # Check if there is value for crawlerRunConfig.proxy_config set add that to context
+            if crawlerRunConfig.proxy_config:
+                proxy_settings = {
+                    "server": crawlerRunConfig.proxy_config.server,
+                }
+                if crawlerRunConfig.proxy_config.username:
+                    proxy_settings.update({
+                        "username": crawlerRunConfig.proxy_config.username,
+                        "password": crawlerRunConfig.proxy_config.password,
+                    })
+                context_settings["proxy"] = proxy_settings
+                
+        # Create and return the context
+        try:
+            # Create the context with appropriate settings
+            context = await self.browser.new_context(**context_settings)
+            
+            # Apply text mode resource blocking if enabled
+            if self.config.text_mode:
+                # Create and apply route patterns for each extension
+                for ext in blocked_extensions:
+                    await context.route(f"**/*.{ext}", lambda route: route.abort())
+                    
+            return context
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error creating browser context: {str(e)}", tag="BROWSER")
+            # Fallback to basic context creation if the advanced settings fail
+            return await self.browser.new_context()
+        
+    async def setup_context(self, context: BrowserContext, crawlerRunConfig: Optional[CrawlerRunConfig] = None):
+        """Set up a browser context with the configured options.
+        
+        Args:
+            context: The browser context to set up
+            crawlerRunConfig: Configuration object containing all browser settings
+        """
+        # Set HTTP headers
+        if self.config.headers:
+            await context.set_extra_http_headers(self.config.headers)
+
+        # Add cookies
+        if self.config.cookies:
+            await context.add_cookies(self.config.cookies)
+
+        # Apply storage state if provided
+        if self.config.storage_state:
+            await context.storage_state(path=None)
+
+        # Configure downloads
+        if self.config.accept_downloads:
+            context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            if self.config.downloads_path:
+                context._impl_obj._options["accept_downloads"] = True
+                context._impl_obj._options["downloads_path"] = self.config.downloads_path
+
+        # Handle user agent and browser hints
+        if self.config.user_agent:
+            combined_headers = {
+                "User-Agent": self.config.user_agent,
+                "sec-ch-ua": self.config.browser_hint,
+            }
+            combined_headers.update(self.config.headers)
+            await context.set_extra_http_headers(combined_headers)
+
+        # Add default cookie
+        target_url = (crawlerRunConfig and crawlerRunConfig.url) or "https://crawl4ai.com/"
+        await context.add_cookies(
+            [
+                {
+                    "name": "cookiesEnabled",
+                    "value": "true",
+                    "url": target_url,
+                }
+            ]
+        )
+
+        # Handle navigator overrides
+        if crawlerRunConfig:
+            if (
+                crawlerRunConfig.override_navigator
+                or crawlerRunConfig.simulate_user
+                or crawlerRunConfig.magic
+            ):
+                await context.add_init_script(load_js_script("navigator_overrider"))
+                
+    async def kill_session(self, session_id: str):
+        """Kill a browser session and clean up resources.
+
+        Args:
+            session_id (str): The session ID to kill.
+        """
+        if session_id not in self.sessions:
+            return
+            
+        context, page, _ = self.sessions[session_id]
+        
+        # Close the page
+        try:
+            await page.close()
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error closing page for session {session_id}: {str(e)}", tag="BROWSER")
+        
+        # Remove session from tracking
+        del self.sessions[session_id]
+        
+        # Clean up any contexts that no longer have pages
+        await self._cleanup_unused_contexts()
+        
+        if self.logger:
+            self.logger.debug(f"Killed session: {session_id}", tag="BROWSER")
+
+    async def _cleanup_unused_contexts(self):
+        """Clean up contexts that no longer have any pages."""
+        async with self._contexts_lock:
+            # Get all contexts we're managing
+            contexts_to_check = list(self.contexts_by_config.values())
+            
+            for context in contexts_to_check:
+                # Check if the context has any pages left
+                if not context.pages:
+                    # No pages left, we can close this context
+                    config_signature = next((sig for sig, ctx in self.contexts_by_config.items() 
+                                           if ctx == context), None)
+                    if config_signature:
+                        try:
+                            await context.close()
+                            del self.contexts_by_config[config_signature]
+                            if self.logger:
+                                self.logger.debug(f"Closed unused context", tag="BROWSER")
+                        except Exception as e:
+                            if self.logger:
+                                self.logger.error(f"Error closing unused context: {str(e)}", tag="BROWSER")
+    
+    def _cleanup_expired_sessions(self):
+        """Clean up expired sessions based on TTL."""
+        current_time = time.time()
+        expired_sessions = [
+            sid
+            for sid, (_, _, last_used) in self.sessions.items()
+            if current_time - last_used > self.session_ttl
+        ]
+        
+        for sid in expired_sessions:
+            if self.logger:
+                self.logger.debug(f"Session expired: {sid}", tag="BROWSER")
+            asyncio.create_task(self.kill_session(sid))
+
+    async def close(self):
+        """Close the browser and clean up resources.
+        
+        This method handles common cleanup tasks like:
+        1. Persisting storage state if a user_data_dir is configured
+        2. Closing all sessions
+        3. Closing all browser contexts
+        4. Closing the browser
+        5. Stopping Playwright
+        
+        Child classes should override this method to add their specific cleanup logic,
+        but should call super().close() to ensure common cleanup tasks are performed.
+        """
+        # Set a flag to prevent race conditions during cleanup
+        self.shutting_down = True
+        
+        try:
+            # Add brief delay if configured
+            if self.config.sleep_on_close:
+                await asyncio.sleep(0.5)
+                
+            # Persist storage state if using a user data directory
+            if self.config.user_data_dir and self.browser:
+                for context in self.browser.contexts:
+                    try:
+                        # Ensure the directory exists
+                        storage_dir = os.path.join(self.config.user_data_dir, "Default")
+                        os.makedirs(storage_dir, exist_ok=True)
+                        
+                        # Save storage state
+                        storage_path = os.path.join(storage_dir, "storage_state.json")
+                        await context.storage_state(path=storage_path)
+                        
+                        if self.logger:
+                            self.logger.debug("Storage state persisted before closing browser", tag="BROWSER")
+                    except Exception as e:
+                        if self.logger:
+                            self.logger.warning(
+                                message="Failed to ensure storage persistence: {error}",
+                                tag="BROWSER", 
+                                params={"error": str(e)}
+                            )
+            
+            # Close all active sessions
+            session_ids = list(self.sessions.keys())
+            for session_id in session_ids:
+                await self.kill_session(session_id)
+                
+            # Close all cached contexts
+            for ctx in self.contexts_by_config.values():
+                try:
+                    await ctx.close()
+                except Exception as e:
+                    if self.logger:
+                        self.logger.error(
+                            message="Error closing context: {error}",
+                            tag="BROWSER",
+                            params={"error": str(e)}
+                        )
+            self.contexts_by_config.clear()
+            
+            # Close the browser if it exists
+            if self.browser:
+                await self.browser.close()
+                self.browser = None
+                
+            # Stop playwright
+            if self.playwright:
+                await self.playwright.stop()
+                self.playwright = None
+                
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    message="Error during browser cleanup: {error}",
+                    tag="BROWSER",
+                    params={"error": str(e)}
+                )
+        finally:
+            # Reset shutting down flag
+            self.shutting_down = False
+    
+    

crawl4ai/browser/strategies/builtin.py

@@ -0,0 +1,468 @@
+import asyncio
+import os
+import time
+import json
+import subprocess
+import shutil
+import signal
+from typing import Optional, Dict, Any, Tuple
+
+
+from ...async_logger import AsyncLogger
+from ...async_configs import CrawlerRunConfig
+from playwright.async_api import Page, BrowserContext
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig
+from ...utils import get_home_folder
+from ..utils import get_browser_executable, is_windows, is_browser_running, find_process_by_port, terminate_process
+
+
+from .cdp import CDPBrowserStrategy
+from .base import BaseBrowserStrategy
+
+class BuiltinBrowserStrategy(CDPBrowserStrategy):
+    """Built-in browser strategy.
+    
+    This strategy extends the CDP strategy to use the built-in browser.
+    """
+    
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the built-in browser strategy.
+        
+        Args:
+            config: Browser configuration
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+        self.builtin_browser_dir = os.path.join(get_home_folder(), "builtin-browser") if not self.config.user_data_dir else self.config.user_data_dir
+        self.builtin_config_file = os.path.join(self.builtin_browser_dir, "browser_config.json")
+
+        # Raise error if user data dir is already engaged
+        if self._check_user_dir_is_engaged(self.builtin_browser_dir):
+            raise Exception(f"User data directory {self.builtin_browser_dir} is already engaged by another browser instance.")
+
+        os.makedirs(self.builtin_browser_dir, exist_ok=True)
+    
+    def _check_user_dir_is_engaged(self, user_data_dir: str) -> bool:
+        """Check if the user data directory is already in use.
+        
+        Returns:
+            bool: True if the directory is engaged, False otherwise
+        """
+        # Load browser config file, then iterate in port_map values, check "user_data_dir" key if it matches
+        # the current user data directory
+        if os.path.exists(self.builtin_config_file):
+            try:
+                with open(self.builtin_config_file, 'r') as f:
+                    browser_info_dict = json.load(f)
+                
+                # Check if user data dir is already engaged
+                for port_str, browser_info in browser_info_dict.get("port_map", {}).items():
+                    if browser_info.get("user_data_dir") == user_data_dir:
+                        return True
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(f"Error reading built-in browser config: {str(e)}", tag="BUILTIN")
+        return False
+
+    async def start(self):
+        """Start or connect to the built-in browser.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Initialize Playwright instance via base class method
+        await BaseBrowserStrategy.start(self)
+        
+        try:
+            # Check for existing built-in browser (get_browser_info already checks if running)
+            browser_info = self.get_browser_info()
+            if browser_info:
+                if self.logger:
+                    self.logger.info(f"Using existing built-in browser at {browser_info.get('cdp_url')}", tag="BROWSER")
+                self.config.cdp_url = browser_info.get('cdp_url')
+            else:
+                if self.logger:
+                    self.logger.info("Built-in browser not found, launching new instance...", tag="BROWSER")
+                cdp_url = await self.launch_builtin_browser(
+                    browser_type=self.config.browser_type,
+                    debugging_port=self.config.debugging_port,
+                    headless=self.config.headless,
+                )
+                if not cdp_url:
+                    if self.logger:
+                        self.logger.warning("Failed to launch built-in browser, falling back to regular CDP strategy", tag="BROWSER")
+                    # Call CDP's start but skip BaseBrowserStrategy.start() since we already called it
+                    return await CDPBrowserStrategy.start(self)
+                self.config.cdp_url = cdp_url
+            
+            # Connect to the browser using CDP protocol
+            self.browser = await self.playwright.chromium.connect_over_cdp(self.config.cdp_url)
+            
+            # Get or create default context
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+            else:
+                self.default_context = await self.create_browser_context()
+            
+            await self.setup_context(self.default_context)
+            
+            if self.logger:
+                self.logger.debug(f"Connected to built-in browser at {self.config.cdp_url}", tag="BUILTIN")
+                
+            return self
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Failed to start built-in browser: {str(e)}", tag="BUILTIN")
+            
+            # There is a possibility that at this point I need to clean up some resourece
+            raise
+
+    def _get_builtin_browser_info(cls, debugging_port: int, config_file: str, logger: Optional[AsyncLogger] = None) -> Optional[Dict[str, Any]]:
+        """Get information about the built-in browser for a specific debugging port.
+        
+        Args:
+            debugging_port: The debugging port to look for
+            config_file: Path to the config file
+            logger: Optional logger for recording events
+            
+        Returns:
+            dict: Browser information or None if no running browser is configured for this port
+        """
+        if not os.path.exists(config_file):
+            return None
+            
+        try:
+            with open(config_file, 'r') as f:
+                browser_info_dict = json.load(f)
+            
+            # Get browser info from port map
+            if isinstance(browser_info_dict, dict) and "port_map" in browser_info_dict:
+                port_str = str(debugging_port)
+                if port_str in browser_info_dict["port_map"]:
+                    browser_info = browser_info_dict["port_map"][port_str]
+                    
+                    # Check if the browser is still running
+                    pids = browser_info.get('pid', '')
+                    if isinstance(pids, str):
+                        pids = [int(pid) for pid in pids.split() if pid.isdigit()]
+                    elif isinstance(pids, int):
+                        pids = [pids]
+                    else:
+                        pids = []
+
+                    # Check if any of the PIDs are running
+                    if not pids:
+                        if logger:
+                            logger.warning(f"Built-in browser on port {debugging_port} has no valid PID", tag="BUILTIN")
+                        # Remove this port from the dictionary
+                        del browser_info_dict["port_map"][port_str]
+                        with open(config_file, 'w') as f:
+                            json.dump(browser_info_dict, f, indent=2)
+                        return None
+                    # Check if any of the PIDs are running
+                    for pid in pids:
+                        if is_browser_running(pid):
+                            browser_info['pid'] = pid
+                            break
+                    else:
+                        # If none of the PIDs are running, remove this port from the dictionary
+                        if logger:
+                            logger.warning(f"Built-in browser on port {debugging_port} is not running", tag="BUILTIN")
+                        # Remove this port from the dictionary
+                        del browser_info_dict["port_map"][port_str]
+                        with open(config_file, 'w') as f:
+                            json.dump(browser_info_dict, f, indent=2)
+                        return None
+                    
+                    return browser_info
+            
+            return None
+                
+        except Exception as e:
+            if logger:
+                logger.error(f"Error reading built-in browser config: {str(e)}", tag="BUILTIN")
+            return None
+            
+    def get_browser_info(self) -> Optional[Dict[str, Any]]:
+        """Get information about the current built-in browser instance.
+        
+        Returns:
+            dict: Browser information or None if no running browser is configured
+        """
+        return self._get_builtin_browser_info(
+            debugging_port=self.config.debugging_port,
+            config_file=self.builtin_config_file,
+            logger=self.logger
+        )
+    
+    async def launch_builtin_browser(self, 
+                               browser_type: str = "chromium",
+                               debugging_port: int = 9222,
+                               headless: bool = True) -> Optional[str]:
+        """Launch a browser in the background for use as the built-in browser.
+        
+        Args:
+            browser_type: Type of browser to launch ('chromium' or 'firefox')
+            debugging_port: Port to use for CDP debugging
+            headless: Whether to run in headless mode
+            
+        Returns:
+            str: CDP URL for the browser, or None if launch failed
+        """
+        # Check if there's an existing browser still running
+        browser_info = self._get_builtin_browser_info(
+            debugging_port=debugging_port,
+            config_file=self.builtin_config_file,
+            logger=self.logger
+        )
+        if browser_info:
+            if self.logger:
+                self.logger.info(f"Built-in browser is already running on port {debugging_port}", tag="BUILTIN")
+            return browser_info.get('cdp_url')
+        
+        # Create a user data directory for the built-in browser
+        user_data_dir = os.path.join(self.builtin_browser_dir, "user_data")
+     
+        # Raise error if user data dir is already engaged
+        if self._check_user_dir_is_engaged(user_data_dir):
+            raise Exception(f"User data directory {user_data_dir} is already engaged by another browser instance.")
+            
+        # Create the user data directory if it doesn't exist
+        os.makedirs(user_data_dir, exist_ok=True)
+        
+        # Prepare browser launch arguments
+        browser_args = super()._build_browser_args()
+        browser_path = await get_browser_executable(browser_type)
+        base_args = [browser_path]
+
+        if browser_type == "chromium":
+            args = [
+                browser_path,
+                f"--remote-debugging-port={debugging_port}",
+                f"--user-data-dir={user_data_dir}",
+            ]
+            # if headless:
+            #     args.append("--headless=new")
+
+        elif browser_type == "firefox":
+            args = [
+                browser_path,
+                "--remote-debugging-port",
+                str(debugging_port),
+                "--profile",
+                user_data_dir,
+            ]
+            if headless:
+                args.append("--headless")
+        else:
+            if self.logger:
+                self.logger.error(f"Browser type {browser_type} not supported for built-in browser", tag="BUILTIN")
+            return None
+        
+        args = base_args + browser_args + args
+        
+        try:
+
+            # Check if the port is already in use
+            PID = ""
+            cdp_url = f"http://localhost:{debugging_port}"
+            config_json = await self._check_port_in_use(cdp_url)
+            if config_json:
+                if self.logger:
+                    self.logger.info(f"Port {debugging_port} is already in use.", tag="BUILTIN")
+                PID = find_process_by_port(debugging_port)
+            else:
+                # Start the browser process detached
+                process = None
+                if is_windows():
+                    process = subprocess.Popen(
+                        args, 
+                        stdout=subprocess.PIPE, 
+                        stderr=subprocess.PIPE,
+                        creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
+                    )
+                else:
+                    process = subprocess.Popen(
+                        args, 
+                        stdout=subprocess.PIPE, 
+                        stderr=subprocess.PIPE,
+                        preexec_fn=os.setpgrp  # Start in a new process group
+                    )
+                
+                # Wait briefly to ensure the process starts successfully
+                await asyncio.sleep(2.0)
+                
+                # Check if the process is still running
+                if process and process.poll() is not None:
+                    if self.logger:
+                        self.logger.error(f"Browser process exited immediately with code {process.returncode}", tag="BUILTIN")
+                    return None
+            
+                PID = process.pid
+                # Construct CDP URL
+                config_json = await self._check_port_in_use(cdp_url)
+
+            
+            # Create browser info
+            browser_info = {
+                'pid': PID,
+                'cdp_url': cdp_url,
+                'user_data_dir': user_data_dir,
+                'browser_type': browser_type,
+                'debugging_port': debugging_port,
+                'start_time': time.time(),
+                'config': config_json
+            }
+            
+            # Read existing config file if it exists
+            port_map = {}
+            if os.path.exists(self.builtin_config_file):
+                try:
+                    with open(self.builtin_config_file, 'r') as f:
+                        existing_data = json.load(f)
+                    
+                    # Check if it already uses port mapping
+                    if isinstance(existing_data, dict) and "port_map" in existing_data:
+                        port_map = existing_data["port_map"]
+
+                    # # Convert legacy format to port mapping
+                    # elif isinstance(existing_data, dict) and "debugging_port" in existing_data:
+                    #     old_port = str(existing_data.get("debugging_port"))
+                    #     if self._is_browser_running(existing_data.get("pid")):
+                    #         port_map[old_port] = existing_data
+                except Exception as e:
+                    if self.logger:
+                        self.logger.warning(f"Could not read existing config: {str(e)}", tag="BUILTIN")
+            
+            # Add/update this browser in the port map
+            port_map[str(debugging_port)] = browser_info
+            
+            # Write updated config
+            with open(self.builtin_config_file, 'w') as f:
+                json.dump({"port_map": port_map}, f, indent=2)
+                
+            # Detach from the browser process - don't keep any references
+            # This is important to allow the Python script to exit while the browser continues running
+            process = None
+                
+            if self.logger:
+                self.logger.success(f"Built-in browser launched at CDP URL: {cdp_url}", tag="BUILTIN")
+            return cdp_url
+            
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error launching built-in browser: {str(e)}", tag="BUILTIN")
+            return None
+
+    async def _check_port_in_use(self, cdp_url: str) -> dict:
+        """Check if a port is already in use by a Chrome DevTools instance.
+        
+        Args:
+            cdp_url: The CDP URL to check
+            
+        Returns:
+            dict: Chrome DevTools protocol version information or None if not found
+        """
+        import aiohttp
+        json_url = f"{cdp_url}/json/version"
+        json_config = None
+        
+        try:
+            async with aiohttp.ClientSession() as session:
+                try:
+                    async with session.get(json_url, timeout=2.0) as response:
+                        if response.status == 200:
+                            json_config = await response.json()
+                            if self.logger:
+                                self.logger.debug(f"Found CDP server running at {cdp_url}", tag="BUILTIN")
+                            return json_config
+                except (aiohttp.ClientError, asyncio.TimeoutError):
+                    pass
+            return None
+        except Exception as e:
+            if self.logger:
+                self.logger.debug(f"Error checking CDP port: {str(e)}", tag="BUILTIN")
+            return None
+
+    async def kill_builtin_browser(self) -> bool:
+        """Kill the built-in browser if it's running.
+        
+        Returns:
+            bool: True if the browser was killed, False otherwise
+        """
+        browser_info = self.get_browser_info()
+        if not browser_info:
+            if self.logger:
+                self.logger.warning(f"No built-in browser found on port {self.config.debugging_port}", tag="BUILTIN")
+            return False
+            
+        pid = browser_info.get('pid')
+        if not pid:
+            return False
+            
+        success, error_msg = terminate_process(pid, logger=self.logger)
+        if success:
+            # Update config file to remove this browser
+            with open(self.builtin_config_file, 'r') as f:
+                browser_info_dict = json.load(f)
+
+            # Remove this port from the dictionary
+            port_str = str(self.config.debugging_port)
+            if port_str in browser_info_dict.get("port_map", {}):
+                del browser_info_dict["port_map"][port_str]
+
+            with open(self.builtin_config_file, 'w') as f:
+                json.dump(browser_info_dict, f, indent=2)
+
+            # Remove user data directory if it exists
+            if os.path.exists(self.builtin_browser_dir):
+                shutil.rmtree(self.builtin_browser_dir)
+
+            # Clear the browser info cache
+            self.browser = None
+            self.temp_dir = None
+            self.shutting_down = True
+                
+            if self.logger:
+                self.logger.success("Built-in browser terminated", tag="BUILTIN")
+            return True
+        else:
+            if self.logger:
+                self.logger.error(f"Error killing built-in browser: {error_msg}", tag="BUILTIN")
+            return False
+    
+    async def get_builtin_browser_status(self) -> Dict[str, Any]:
+        """Get status information about the built-in browser.
+        
+        Returns:
+            dict: Status information with running, cdp_url, and info fields
+        """
+        browser_info = self.get_browser_info()
+        
+        if not browser_info:
+            return {
+                'running': False,
+                'cdp_url': None,
+                'info': None,
+                'port': self.config.debugging_port
+            }
+            
+        return {
+            'running': True,
+            'cdp_url': browser_info.get('cdp_url'),
+            'info': browser_info,
+            'port': self.config.debugging_port
+        }
+
+    async def close(self):
+        """Close the built-in browser and clean up resources."""
+        # Call parent class close method
+        await super().close()
+        
+        # Clean up built-in browser if we created it and were in shutdown mode
+        if self.shutting_down:
+            await self.kill_builtin_browser()
+            if self.logger:
+                self.logger.debug("Killed built-in browser during shutdown", tag="BUILTIN")

crawl4ai/browser/strategies/cdp.py

@@ -0,0 +1,281 @@
+"""Browser strategies module for Crawl4AI.
+
+This module implements the browser strategy pattern for different
+browser implementations, including Playwright, CDP, and builtin browsers.
+"""
+
+import asyncio
+import os
+import time
+import json
+import subprocess
+import shutil
+from typing import Optional, Tuple, List
+
+from playwright.async_api import BrowserContext, Page
+
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig, CrawlerRunConfig
+from ..utils import get_playwright, get_browser_executable, create_temp_directory, is_windows, check_process_is_running, terminate_process
+
+from .base import BaseBrowserStrategy
+
+class CDPBrowserStrategy(BaseBrowserStrategy):
+    """CDP-based browser strategy.
+    
+    This strategy connects to an existing browser using CDP protocol or
+    launches and connects to a browser using CDP.
+    """
+    
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the CDP browser strategy.
+        
+        Args:
+            config: Browser configuration
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+        self.browser_process = None
+        self.temp_dir = None
+        self.shutting_down = False
+        
+    async def start(self):
+        """Start or connect to the browser using CDP.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Call the base class start to initialize Playwright
+        await super().start()
+        
+        try:
+            # Get or create CDP URL
+            cdp_url = await self._get_or_create_cdp_url()
+            
+            # Connect to the browser using CDP
+            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
+            
+            # Get or create default context
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+            else:
+                self.default_context = await self.create_browser_context()
+            
+            await self.setup_context(self.default_context)
+            
+            if self.logger:
+                self.logger.debug(f"Connected to CDP browser at {cdp_url}", tag="CDP")
+
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Failed to connect to CDP browser: {str(e)}", tag="CDP")
+
+            # Clean up any resources before re-raising
+            await self._cleanup_process()
+            raise
+            
+        return self
+    
+    async def _get_or_create_cdp_url(self) -> str:
+        """Get existing CDP URL or launch a browser and return its CDP URL.
+        
+        Returns:
+            str: CDP URL for connecting to the browser
+        """
+        # If CDP URL is provided, just return it
+        if self.config.cdp_url:
+            return self.config.cdp_url
+
+        # Create temp dir if needed
+        if not self.config.user_data_dir:
+            self.temp_dir = create_temp_directory()
+            user_data_dir = self.temp_dir
+        else:
+            user_data_dir = self.config.user_data_dir
+
+        # Get browser args based on OS and browser type
+        # args = await self._get_browser_args(user_data_dir)
+        browser_args = super()._build_browser_args()
+        browser_path = await get_browser_executable(self.config.browser_type)
+        base_args = [browser_path]
+
+        if self.config.browser_type == "chromium":
+            args = [
+                f"--remote-debugging-port={self.config.debugging_port}",
+                f"--user-data-dir={user_data_dir}",
+            ]
+            # if self.config.headless:
+            #     args.append("--headless=new")
+
+        elif self.config.browser_type == "firefox":
+            args = [
+                "--remote-debugging-port",
+                str(self.config.debugging_port),
+                "--profile",
+                user_data_dir,
+            ]
+            if self.config.headless:
+                args.append("--headless")
+        else:
+            raise NotImplementedError(f"Browser type {self.config.browser_type} not supported")
+
+        args = base_args + browser_args['args'] + args         
+
+        # Start browser process
+        try:
+            # Use DETACHED_PROCESS flag on Windows to fully detach the process
+            # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group
+            if is_windows():
+                self.browser_process = subprocess.Popen(
+                    args, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
+                )
+            else:
+                self.browser_process = subprocess.Popen(
+                    args, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    preexec_fn=os.setpgrp  # Start in a new process group
+                )
+                
+            # Monitor for a short time to make sure it starts properly
+            is_running, return_code, stdout, stderr = await check_process_is_running(self.browser_process, delay=2)
+            if not is_running:
+                if self.logger:
+                    self.logger.error(
+                        message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
+                        tag="ERROR",
+                        params={
+                            "code": return_code,
+                            "stdout": stdout.decode() if stdout else "",
+                            "stderr": stderr.decode() if stderr else "",
+                        },
+                    )
+                await self._cleanup_process()
+                raise Exception("Browser process terminated unexpectedly")
+
+            return f"http://localhost:{self.config.debugging_port}"
+        except Exception as e:
+            await self._cleanup_process()
+            raise Exception(f"Failed to start browser: {e}")    
+
+    async def _cleanup_process(self):
+        """Cleanup browser process and temporary directory."""
+        # Set shutting_down flag BEFORE any termination actions
+        self.shutting_down = True
+
+        if self.browser_process:
+            try:
+                # Only attempt termination if the process is still running
+                if self.browser_process.poll() is None:
+                    # Use our robust cross-platform termination utility
+                    success = terminate_process(
+                        pid=self.browser_process.pid,
+                        timeout=1.0,  # Equivalent to the previous 10*0.1s wait
+                        logger=self.logger
+                    )
+                    
+                    if not success and self.logger:
+                        self.logger.warning(
+                            message="Failed to terminate browser process cleanly",
+                            tag="PROCESS"
+                        )
+                        
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(
+                        message="Error during browser process cleanup: {error}",
+                        tag="ERROR",
+                        params={"error": str(e)},
+                    )
+
+        if self.temp_dir and os.path.exists(self.temp_dir):
+            try:
+                shutil.rmtree(self.temp_dir)
+                self.temp_dir = None
+                if self.logger:
+                    self.logger.debug("Removed temporary directory", tag="CDP")
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(
+                        message="Error removing temporary directory: {error}",
+                        tag="CDP",
+                        params={"error": str(e)}
+                    )
+
+        self.browser_process = None
+    
+    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        # For CDP, we typically use the shared default_context
+        context = self.default_context
+        pages = context.pages
+        
+        # Otherwise, check if we have an existing context for this config
+        config_signature = self._make_config_signature(crawlerRunConfig)
+        self.contexts_by_config[config_signature] = context
+
+        await self.setup_context(context, crawlerRunConfig)
+
+        # Check if there's already a page with the target URL
+        page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+        
+        # If not found, create a new page
+        if not page:
+            page = await context.new_page()
+        
+        return page, context
+
+    async def _get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page for the given configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Call parent method to ensure browser is started
+        await super().get_page(crawlerRunConfig)
+        
+        # For CDP, we typically use the shared default_context
+        context = self.default_context
+        pages = context.pages
+        
+        # Otherwise, check if we have an existing context for this config
+        config_signature = self._make_config_signature(crawlerRunConfig)
+        self.contexts_by_config[config_signature] = context
+
+        await self.setup_context(context, crawlerRunConfig)
+
+        # Check if there's already a page with the target URL
+        page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+        
+        # If not found, create a new page
+        if not page:
+            page = await context.new_page()
+        
+        # If a session_id is specified, store this session for reuse
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+        
+        return page, context
+
+    async def close(self):
+        """Close the CDP browser and clean up resources."""
+        # Skip cleanup if using external CDP URL and not launched by us
+        if self.config.cdp_url and not self.browser_process:
+            if self.logger:
+                self.logger.debug("Skipping cleanup for external CDP browser", tag="CDP")
+            return
+        
+        # Call parent implementation for common cleanup
+        await super().close()
+        
+        # Additional CDP-specific cleanup
+        await asyncio.sleep(0.5)
+        await self._cleanup_process()

crawl4ai/browser/strategies/docker_strategy.py

@@ -0,0 +1,430 @@
+"""Docker browser strategy module for Crawl4AI.
+
+This module provides browser strategies for running browsers in Docker containers,
+which offers better isolation, consistency across platforms, and easy scaling.
+"""
+
+import os
+import uuid
+from typing import List, Optional
+
+
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig
+from ..models import DockerConfig
+from ..docker_registry import DockerRegistry
+from ..docker_utils import DockerUtils
+from .builtin import CDPBrowserStrategy
+from .base import BaseBrowserStrategy
+
+class DockerBrowserStrategy(CDPBrowserStrategy):
+    """Docker-based browser strategy.
+
+    Extends the CDPBrowserStrategy to run browsers in Docker containers.
+    Supports two modes:
+    1. "connect" - Uses a Docker image with Chrome already running
+    2. "launch" - Starts Chrome within the container with custom settings
+
+    Attributes:
+        docker_config: Docker-specific configuration options
+        container_id: ID of current Docker container
+        container_name: Name assigned to the container
+        registry: Registry for tracking and reusing containers
+        docker_utils: Utilities for Docker operations
+        chrome_process_id: Process ID of Chrome within container
+        socat_process_id: Process ID of socat within container
+        internal_cdp_port: Chrome's internal CDP port
+        internal_mapped_port: Port that socat maps to internally
+    """
+
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the Docker browser strategy.
+
+        Args:
+            config: Browser configuration including Docker-specific settings
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+
+        # Initialize Docker-specific attributes
+        self.docker_config = self.config.docker_config or DockerConfig()
+        self.container_id = None
+        self.container_name = f"crawl4ai-browser-{uuid.uuid4().hex[:8]}"
+
+        # Use the shared registry file path for consistency with BuiltinBrowserStrategy
+        registry_file = self.docker_config.registry_file
+        if registry_file is None and self.config.user_data_dir:
+            # Use the same registry file as BuiltinBrowserStrategy if possible
+            registry_file = os.path.join(
+                os.path.dirname(self.config.user_data_dir), "browser_config.json"
+            )
+
+        self.registry = DockerRegistry(self.docker_config.registry_file)
+        self.docker_utils = DockerUtils(logger)
+        self.chrome_process_id = None
+        self.socat_process_id = None
+        self.internal_cdp_port = 9222  # Chrome's internal CDP port
+        self.internal_mapped_port = 9223  # Port that socat maps to internally
+        self.shutting_down = False
+
+    async def start(self):
+        """Start or connect to a browser running in a Docker container.
+
+        This method initializes Playwright and establishes a connection to
+        a browser running in a Docker container. Depending on the configured mode:
+        - "connect": Connects to a container with Chrome already running
+        - "launch": Creates a container and launches Chrome within it
+
+        Returns:
+            self: For method chaining
+        """
+        # Initialize Playwright
+        await BaseBrowserStrategy.start(self)
+
+        if self.logger:
+            self.logger.info(
+                f"Starting Docker browser strategy in {self.docker_config.mode} mode",
+                tag="DOCKER",
+            )
+
+        try:
+            # Get CDP URL by creating or reusing a Docker container
+            # This handles the container management and browser startup
+            cdp_url = await self._get_or_create_cdp_url()
+
+            if not cdp_url:
+                raise Exception(
+                    "Failed to establish CDP connection to Docker container"
+                )
+
+            if self.logger:
+                self.logger.info(
+                    f"Connecting to browser in Docker via CDP: {cdp_url}", tag="DOCKER"
+                )
+
+            # Connect to the browser using CDP
+            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
+
+            # Get existing context or create default context
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+                if self.logger:
+                    self.logger.debug("Using existing browser context", tag="DOCKER")
+            else:
+                if self.logger:
+                    self.logger.debug("Creating new browser context", tag="DOCKER")
+                self.default_context = await self.create_browser_context()
+                await self.setup_context(self.default_context)
+
+            return self
+
+        except Exception as e:
+            # Clean up resources if startup fails
+            if self.container_id and not self.docker_config.persistent:
+                if self.logger:
+                    self.logger.warning(
+                        f"Cleaning up container after failed start: {self.container_id[:12]}",
+                        tag="DOCKER",
+                    )
+                await self.docker_utils.remove_container(self.container_id)
+                self.registry.unregister_container(self.container_id)
+                self.container_id = None
+
+            if self.playwright:
+                await self.playwright.stop()
+                self.playwright = None
+
+            # Re-raise the exception
+            if self.logger:
+                self.logger.error(
+                    f"Failed to start Docker browser: {str(e)}", tag="DOCKER"
+                )
+            raise
+
+    async def _generate_config_hash(self) -> str:
+        """Generate a hash of the configuration for container matching.
+
+        Returns:
+            Hash string uniquely identifying this configuration
+        """
+        # Create a dict with the relevant parts of the config
+        config_dict = {
+            "image": self.docker_config.image,
+            "mode": self.docker_config.mode,
+            "browser_type": self.config.browser_type,
+            "headless": self.config.headless,
+        }
+
+        # Add browser-specific config if in launch mode
+        if self.docker_config.mode == "launch":
+            config_dict.update(
+                {
+                    "text_mode": self.config.text_mode,
+                    "light_mode": self.config.light_mode,
+                    "viewport_width": self.config.viewport_width,
+                    "viewport_height": self.config.viewport_height,
+                }
+            )
+
+        # Use the utility method to generate the hash
+        return self.docker_utils.generate_config_hash(config_dict)
+
+    async def _get_or_create_cdp_url(self) -> str:
+        """Get CDP URL by either creating a new container or using an existing one.
+
+        Returns:
+            CDP URL for connecting to the browser
+
+        Raises:
+            Exception: If container creation or browser launch fails
+        """
+        # If CDP URL is explicitly provided, use it
+        if self.config.cdp_url:
+            return self.config.cdp_url
+
+        # Ensure Docker image exists (will build if needed)
+        image_name = await self.docker_utils.ensure_docker_image_exists(
+            self.docker_config.image, self.docker_config.mode
+        )
+
+        # Generate config hash for container matching
+        config_hash = await self._generate_config_hash()
+
+        # Look for existing container with matching config
+        container_id = await self.registry.find_container_by_config(
+            config_hash, self.docker_utils
+        )
+
+        if container_id:
+            # Use existing container
+            self.container_id = container_id
+            host_port = self.registry.get_container_host_port(container_id)
+            if self.logger:
+                self.logger.info(
+                    f"Using existing Docker container: {container_id[:12]}",
+                    tag="DOCKER",
+                )
+        else:
+            # Get a port for the new container
+            host_port = (
+                self.docker_config.host_port
+                or self.registry.get_next_available_port(self.docker_utils)
+            )
+
+            # Prepare volumes list
+            volumes = list(self.docker_config.volumes)
+
+            # Add user data directory if specified
+            if self.docker_config.user_data_dir:
+                # Ensure user data directory exists
+                os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
+                volumes.append(
+                    f"{self.docker_config.user_data_dir}:{self.docker_config.container_user_data_dir}"
+                )
+
+                # # Update config user_data_dir to point to container path
+                # self.config.user_data_dir = self.docker_config.container_user_data_dir
+
+            # Create a new container
+            container_id = await self.docker_utils.create_container(
+                image_name=image_name,
+                host_port=host_port,
+                container_name=self.container_name,
+                volumes=volumes,
+                network=self.docker_config.network,
+                env_vars=self.docker_config.env_vars,
+                cpu_limit=self.docker_config.cpu_limit,
+                memory_limit=self.docker_config.memory_limit,
+                extra_args=self.docker_config.extra_args,
+            )
+
+            if not container_id:
+                raise Exception("Failed to create Docker container")
+
+            self.container_id = container_id
+
+            # Wait for container to be ready
+            await self.docker_utils.wait_for_container_ready(container_id)
+
+            # Handle specific setup based on mode
+            if self.docker_config.mode == "launch":
+                # In launch mode, we need to start socat and Chrome
+                await self.docker_utils.start_socat_in_container(container_id)
+
+                # Build browser arguments
+                browser_args = self._build_browser_args()
+
+                # Launch Chrome
+                await self.docker_utils.launch_chrome_in_container(
+                    container_id, browser_args
+                )
+
+                # Get PIDs for later cleanup
+                self.chrome_process_id = (
+                    await self.docker_utils.get_process_id_in_container(
+                        container_id, "chromium"
+                    )
+                )
+                self.socat_process_id = (
+                    await self.docker_utils.get_process_id_in_container(
+                        container_id, "socat"
+                    )
+                )
+
+            # Wait for CDP to be ready
+            cdp_json_config = await self.docker_utils.wait_for_cdp_ready(host_port)
+
+            if cdp_json_config:
+                # Register the container in the shared registry
+                self.registry.register_container(
+                    container_id, host_port, config_hash, cdp_json_config
+                )
+            else:
+                raise Exception("Failed to get CDP JSON config from Docker container")
+
+            if self.logger:
+                self.logger.success(
+                    f"Docker container ready: {container_id[:12]} on port {host_port}",
+                    tag="DOCKER",
+                )
+
+        # Return CDP URL
+        return f"http://localhost:{host_port}"
+
+    def _build_browser_args(self) -> List[str]:
+        """Build Chrome command line arguments based on BrowserConfig.
+
+        Returns:
+            List of command line arguments for Chrome
+        """
+        # Call parent method to get common arguments
+        browser_args = super()._build_browser_args()
+        return browser_args["args"] + [
+            f"--remote-debugging-port={self.internal_cdp_port}",
+            "--remote-debugging-address=0.0.0.0",  # Allow external connections
+            "--disable-dev-shm-usage",
+            "--headless=new",
+        ]
+
+        # args = [
+        #     "--no-sandbox",
+        #     "--disable-gpu",
+        #     f"--remote-debugging-port={self.internal_cdp_port}",
+        #     "--remote-debugging-address=0.0.0.0",  # Allow external connections
+        #     "--disable-dev-shm-usage",
+        # ]
+
+        # if self.config.headless:
+        #     args.append("--headless=new")
+
+        # if self.config.viewport_width and self.config.viewport_height:
+        #     args.append(f"--window-size={self.config.viewport_width},{self.config.viewport_height}")
+
+        # if self.config.user_agent:
+        #     args.append(f"--user-agent={self.config.user_agent}")
+
+        # if self.config.text_mode:
+        #     args.extend([
+        #         "--blink-settings=imagesEnabled=false",
+        #         "--disable-remote-fonts",
+        #         "--disable-images",
+        #         "--disable-javascript",
+        #     ])
+
+        # if self.config.light_mode:
+        #     # Import here to avoid circular import
+        #     from ..utils import get_browser_disable_options
+        #     args.extend(get_browser_disable_options())
+
+        # if self.config.user_data_dir:
+        #     args.append(f"--user-data-dir={self.config.user_data_dir}")
+
+        # if self.config.extra_args:
+        #     args.extend(self.config.extra_args)
+
+        # return args
+
+    async def close(self):
+        """Close the browser and clean up Docker container if needed."""
+        # Set flag to track if we were the ones initiating shutdown
+        initiated_shutdown = not self.shutting_down
+        # Storage persistence for Docker needs special handling
+        # We need to store state before calling super().close() which will close the browser
+        if (
+            self.browser
+            and self.docker_config.user_data_dir
+            and self.docker_config.persistent
+        ):
+            for context in self.browser.contexts:
+                try:
+                    # Ensure directory exists
+                    os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
+
+                    # Save storage state to user data directory
+                    storage_path = os.path.join(
+                        self.docker_config.user_data_dir, "storage_state.json"
+                    )
+                    await context.storage_state(path=storage_path)
+                    if self.logger:
+                        self.logger.debug(
+                            "Persisted Docker-specific storage state", tag="DOCKER"
+                        )
+                except Exception as e:
+                    if self.logger:
+                        self.logger.warning(
+                            message="Failed to persist Docker storage state: {error}",
+                            tag="DOCKER",
+                            params={"error": str(e)},
+                        )
+
+        # Call parent method to handle common cleanup
+        await super().close()
+
+        # Only perform container cleanup if we initiated shutdown
+        # and we need to handle Docker-specific resources
+        if initiated_shutdown:
+            # Only clean up container if not persistent
+            if self.container_id and not self.docker_config.persistent:
+                # Stop Chrome process in "launch" mode
+                if self.docker_config.mode == "launch" and self.chrome_process_id:
+                    await self.docker_utils.stop_process_in_container(
+                        self.container_id, self.chrome_process_id
+                    )
+                    if self.logger:
+                        self.logger.debug(
+                            f"Stopped Chrome process {self.chrome_process_id} in container",
+                            tag="DOCKER",
+                        )
+
+                # Stop socat process in "launch" mode
+                if self.docker_config.mode == "launch" and self.socat_process_id:
+                    await self.docker_utils.stop_process_in_container(
+                        self.container_id, self.socat_process_id
+                    )
+                    if self.logger:
+                        self.logger.debug(
+                            f"Stopped socat process {self.socat_process_id} in container",
+                            tag="DOCKER",
+                        )
+
+                # Remove or stop container based on configuration
+                if self.docker_config.remove_on_exit:
+                    await self.docker_utils.remove_container(self.container_id)
+                    # Unregister from registry
+                    if hasattr(self, "registry") and self.registry:
+                        self.registry.unregister_container(self.container_id)
+                    if self.logger:
+                        self.logger.debug(
+                            f"Removed Docker container {self.container_id}",
+                            tag="DOCKER",
+                        )
+                else:
+                    await self.docker_utils.stop_container(self.container_id)
+                    if self.logger:
+                        self.logger.debug(
+                            f"Stopped Docker container {self.container_id}",
+                            tag="DOCKER",
+                        )
+
+                self.container_id = None

crawl4ai/browser/strategies/playwright.py

@@ -0,0 +1,134 @@
+"""Browser strategies module for Crawl4AI.
+
+This module implements the browser strategy pattern for different
+browser implementations, including Playwright, CDP, and builtin browsers.
+"""
+
+import time
+from typing import Optional, Tuple
+
+from playwright.async_api import BrowserContext, Page
+
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig, CrawlerRunConfig
+
+from playwright_stealth import StealthConfig
+
+from .base import BaseBrowserStrategy
+
+stealth_config = StealthConfig(
+    webdriver=True,
+    chrome_app=True,
+    chrome_csi=True,
+    chrome_load_times=True,
+    chrome_runtime=True,
+    navigator_languages=True,
+    navigator_plugins=True,
+    navigator_permissions=True,
+    webgl_vendor=True,
+    outerdimensions=True,
+    navigator_hardware_concurrency=True,
+    media_codecs=True,
+)
+
+class PlaywrightBrowserStrategy(BaseBrowserStrategy):
+    """Standard Playwright browser strategy.
+    
+    This strategy launches a new browser instance using Playwright
+    and manages browser contexts.
+    """
+    
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the Playwright browser strategy.
+        
+        Args:
+            config: Browser configuration
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+        # No need to re-initialize sessions and session_ttl as they're now in the base class
+    
+    async def start(self):
+        """Start the browser instance.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Call the base class start to initialize Playwright
+        await super().start()
+        
+        # Build browser arguments using the base class method
+        browser_args = self._build_browser_args()
+        
+        try:
+            # Launch appropriate browser type
+            if self.config.browser_type == "firefox":
+                self.browser = await self.playwright.firefox.launch(**browser_args)
+            elif self.config.browser_type == "webkit":
+                self.browser = await self.playwright.webkit.launch(**browser_args)
+            else:
+                self.browser = await self.playwright.chromium.launch(**browser_args)
+                
+            self.default_context = self.browser
+            
+            if self.logger:
+                self.logger.debug(f"Launched {self.config.browser_type} browser", tag="BROWSER")
+                
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Failed to launch browser: {str(e)}", tag="BROWSER")
+            raise
+            
+        return self
+
+    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        # Otherwise, check if we have an existing context for this config
+        config_signature = self._make_config_signature(crawlerRunConfig)
+        
+        async with self._contexts_lock:
+            if config_signature in self.contexts_by_config:
+                context = self.contexts_by_config[config_signature]
+            else:
+                # Create and setup a new context
+                context = await self.create_browser_context(crawlerRunConfig)
+                await self.setup_context(context, crawlerRunConfig)
+                self.contexts_by_config[config_signature] = context
+
+        # Create a new page from the chosen context
+        page = await context.new_page()
+        
+        return page, context
+
+    async def _get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page for the given configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Call parent method to ensure browser is started
+        await super().get_page(crawlerRunConfig)
+        
+        # Otherwise, check if we have an existing context for this config
+        config_signature = self._make_config_signature(crawlerRunConfig)
+        
+        async with self._contexts_lock:
+            if config_signature in self.contexts_by_config:
+                context = self.contexts_by_config[config_signature]
+            else:
+                # Create and setup a new context
+                context = await self.create_browser_context(crawlerRunConfig)
+                await self.setup_context(context, crawlerRunConfig)
+                self.contexts_by_config[config_signature] = context
+
+        # Create a new page from the chosen context
+        page = await context.new_page()
+        
+        # If a session_id is specified, store this session so we can reuse later
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            
+        return page, context
+        

crawl4ai/browser/utils.py

@@ -0,0 +1,465 @@
+"""Browser utilities module for Crawl4AI.
+
+This module provides utility functions for browser management,
+including process management, CDP connection utilities,
+and Playwright instance management.
+"""
+
+import asyncio
+import os
+import sys
+import time
+import tempfile
+import subprocess
+from typing import Optional, Tuple, Union
+import signal
+import psutil
+
+from playwright.async_api import async_playwright
+
+from ..utils import get_chromium_path
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+
+from ..async_logger import AsyncLogger
+
+
+_playwright_instance = None
+
+async def get_playwright():
+    """Get or create the Playwright instance (singleton pattern).
+    
+    Returns:
+        Playwright: The Playwright instance
+    """
+    global _playwright_instance
+    if _playwright_instance is None or True:
+        _playwright_instance = await async_playwright().start()
+    return _playwright_instance
+
+async def get_browser_executable(browser_type: str) -> str:
+    """Get the path to browser executable, with platform-specific handling.
+    
+    Args:
+        browser_type: Type of browser (chromium, firefox, webkit)
+        
+    Returns:
+        Path to browser executable
+    """
+    return await get_chromium_path(browser_type)
+
+def create_temp_directory(prefix="browser-profile-") -> str:
+    """Create a temporary directory for browser data.
+    
+    Args:
+        prefix: Prefix for the temporary directory name
+        
+    Returns:
+        Path to the created temporary directory
+    """
+    return tempfile.mkdtemp(prefix=prefix)
+
+def is_windows() -> bool:
+    """Check if the current platform is Windows.
+    
+    Returns:
+        True if Windows, False otherwise
+    """
+    return sys.platform == "win32"
+
+def is_macos() -> bool:
+    """Check if the current platform is macOS.
+    
+    Returns:
+        True if macOS, False otherwise
+    """
+    return sys.platform == "darwin"
+
+def is_linux() -> bool:
+    """Check if the current platform is Linux.
+    
+    Returns:
+        True if Linux, False otherwise
+    """
+    return not (is_windows() or is_macos())
+    
+def is_browser_running(pid: Optional[int]) -> bool:
+    """Check if a process with the given PID is running.
+    
+    Args:
+        pid: Process ID to check
+        
+    Returns:
+        bool: True if the process is running, False otherwise
+    """
+    if not pid:
+        return False
+        
+    try:
+        if type(pid) is str:
+            pid = int(pid)
+        # Check if the process exists
+        if is_windows():
+            process = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], 
+                                     capture_output=True, text=True)
+            return str(pid) in process.stdout
+        else:
+            # Unix-like systems
+            os.kill(pid, 0)  # This doesn't actually kill the process, just checks if it exists
+        return True
+    except (ProcessLookupError, PermissionError, OSError):
+        return False
+
+def get_browser_disable_options() -> list:
+    """Get standard list of browser disable options for performance.
+    
+    Returns:
+        List of command-line options to disable various browser features
+    """
+    return [
+        "--disable-background-networking",
+        "--disable-background-timer-throttling",
+        "--disable-backgrounding-occluded-windows",
+        "--disable-breakpad",
+        "--disable-client-side-phishing-detection",
+        "--disable-component-extensions-with-background-pages",
+        "--disable-default-apps",
+        "--disable-extensions",
+        "--disable-features=TranslateUI",
+        "--disable-hang-monitor",
+        "--disable-ipc-flooding-protection",
+        "--disable-popup-blocking",
+        "--disable-prompt-on-repost",
+        "--disable-sync",
+        "--force-color-profile=srgb",
+        "--metrics-recording-only",
+        "--no-first-run",
+        "--password-store=basic",
+        "--use-mock-keychain",
+    ]
+
+
+async def find_optimal_browser_config(total_urls=50, verbose=True, rate_limit_delay=0.2):
+    """Find optimal browser configuration for crawling a specific number of URLs.
+    
+    Args:
+        total_urls: Number of URLs to crawl
+        verbose: Whether to print progress
+        rate_limit_delay: Delay between page loads to avoid rate limiting
+        
+    Returns:
+        dict: Contains fastest, lowest_memory, and optimal configurations
+    """
+    from .manager import BrowserManager
+    if verbose:
+        print(f"\n=== Finding optimal configuration for crawling {total_urls} URLs ===\n")
+    
+    # Generate test URLs with timestamp to avoid caching
+    timestamp = int(time.time())
+    urls = [f"https://example.com/page_{i}?t={timestamp}" for i in range(total_urls)]
+    
+    # Limit browser configurations to test (1 browser to max 10)
+    max_browsers = min(10, total_urls)
+    configs_to_test = []
+    
+    # Generate configurations (browser count, pages distribution)
+    for num_browsers in range(1, max_browsers + 1):
+        base_pages = total_urls // num_browsers
+        remainder = total_urls % num_browsers
+        
+        # Create distribution array like [3, 3, 2, 2] (some browsers get one more page)
+        if remainder > 0:
+            distribution = [base_pages + 1] * remainder + [base_pages] * (num_browsers - remainder)
+        else:
+            distribution = [base_pages] * num_browsers
+            
+        configs_to_test.append((num_browsers, distribution))
+    
+    results = []
+    
+    # Test each configuration
+    for browser_count, page_distribution in configs_to_test:
+        if verbose:
+            print(f"Testing {browser_count} browsers with distribution {tuple(page_distribution)}")
+        
+        try:
+            # Track memory if possible
+            try:
+                import psutil
+                process = psutil.Process()
+                start_memory = process.memory_info().rss / (1024 * 1024)  # MB
+            except ImportError:
+                if verbose: 
+                    print("Memory tracking not available (psutil not installed)")
+                start_memory = 0
+            
+            # Start browsers in parallel
+            managers = []
+            start_tasks = []
+            start_time = time.time()
+
+            logger = AsyncLogger(verbose=True, log_file=None)
+            
+            for i in range(browser_count):
+                config = BrowserConfig(headless=True)
+                manager = BrowserManager(browser_config=config, logger=logger)
+                start_tasks.append(manager.start())
+                managers.append(manager)
+            
+            await asyncio.gather(*start_tasks)
+            
+            # Distribute URLs among browsers
+            urls_per_manager = {}
+            url_index = 0
+            
+            for i, manager in enumerate(managers):
+                pages_for_this_browser = page_distribution[i]
+                end_index = url_index + pages_for_this_browser
+                urls_per_manager[manager] = urls[url_index:end_index]
+                url_index = end_index
+            
+            # Create pages for each browser
+            all_pages = []
+            for manager, manager_urls in urls_per_manager.items():
+                if not manager_urls:
+                    continue
+                pages = await manager.get_pages(CrawlerRunConfig(), count=len(manager_urls))
+                all_pages.extend(zip(pages, manager_urls))
+            
+            # Crawl pages with delay to avoid rate limiting
+            async def crawl_page(page_ctx, url):
+                page, _ = page_ctx
+                try:
+                    await page.goto(url)
+                    if rate_limit_delay > 0:
+                        await asyncio.sleep(rate_limit_delay)
+                    title = await page.title()
+                    return title
+                finally:
+                    await page.close()
+            
+            crawl_start = time.time()
+            crawl_tasks = [crawl_page(page_ctx, url) for page_ctx, url in all_pages]
+            await asyncio.gather(*crawl_tasks)
+            crawl_time = time.time() - crawl_start
+            total_time = time.time() - start_time
+            
+            # Measure final memory usage
+            if start_memory > 0:
+                end_memory = process.memory_info().rss / (1024 * 1024)
+                memory_used = end_memory - start_memory
+            else:
+                memory_used = 0
+            
+            # Close all browsers
+            for manager in managers:
+                await manager.close()
+            
+            # Calculate metrics
+            pages_per_second = total_urls / crawl_time
+            
+            # Calculate efficiency score (higher is better)
+            # This balances speed vs memory
+            if memory_used > 0:
+                efficiency = pages_per_second / (memory_used + 1)
+            else:
+                efficiency = pages_per_second
+            
+            # Store result
+            result = {
+                "browser_count": browser_count,
+                "distribution": tuple(page_distribution),
+                "crawl_time": crawl_time,
+                "total_time": total_time,
+                "memory_used": memory_used,
+                "pages_per_second": pages_per_second, 
+                "efficiency": efficiency
+            }
+            
+            results.append(result)
+            
+            if verbose:
+                print(f"  ✓ Crawled {total_urls} pages in {crawl_time:.2f}s ({pages_per_second:.1f} pages/sec)")
+                if memory_used > 0:
+                    print(f"  ✓ Memory used: {memory_used:.1f}MB ({memory_used/total_urls:.1f}MB per page)")
+                print(f"  ✓ Efficiency score: {efficiency:.4f}")
+            
+        except Exception as e:
+            if verbose:
+                print(f"  ✗ Error: {str(e)}")
+            
+            # Clean up
+            for manager in managers:
+                try:
+                    await manager.close()
+                except:
+                    pass
+    
+    # If no successful results, return None
+    if not results:
+        return None
+    
+    # Find best configurations
+    fastest = sorted(results, key=lambda x: x["crawl_time"])[0]
+    
+    # Only consider memory if available
+    memory_results = [r for r in results if r["memory_used"] > 0]
+    if memory_results:
+        lowest_memory = sorted(memory_results, key=lambda x: x["memory_used"])[0]
+    else:
+        lowest_memory = fastest
+    
+    # Find most efficient (balanced speed vs memory)
+    optimal = sorted(results, key=lambda x: x["efficiency"], reverse=True)[0]
+    
+    # Print summary
+    if verbose:
+        print("\n=== OPTIMAL CONFIGURATIONS ===")
+        print(f"⚡ Fastest: {fastest['browser_count']} browsers {fastest['distribution']}")
+        print(f"   {fastest['crawl_time']:.2f}s, {fastest['pages_per_second']:.1f} pages/sec")
+        
+        print(f"💾 Memory-efficient: {lowest_memory['browser_count']} browsers {lowest_memory['distribution']}")
+        if lowest_memory["memory_used"] > 0:
+            print(f"   {lowest_memory['memory_used']:.1f}MB, {lowest_memory['memory_used']/total_urls:.2f}MB per page")
+        
+        print(f"🌟 Balanced optimal: {optimal['browser_count']} browsers {optimal['distribution']}")
+        print(f"   {optimal['crawl_time']:.2f}s, {optimal['pages_per_second']:.1f} pages/sec, score: {optimal['efficiency']:.4f}")
+    
+    return {
+        "fastest": fastest,
+        "lowest_memory": lowest_memory,
+        "optimal": optimal,
+        "all_configs": results
+    }
+
+
+# Find process ID of the existing browser using os
+def find_process_by_port(port: int) -> str:
+    """Find process ID listening on a specific port.
+    
+    Args:
+        port: Port number to check
+        
+    Returns:
+        str: Process ID or empty string if not found
+    """
+    try:
+        if is_windows():
+            cmd = f"netstat -ano | findstr :{port}"
+            result = subprocess.check_output(cmd, shell=True).decode()
+            return result.strip().split()[-1] if result else ""
+        else:
+            cmd = f"lsof -i :{port} -t"
+            return subprocess.check_output(cmd, shell=True).decode().strip()
+    except subprocess.CalledProcessError:
+        return ""
+    
+async def check_process_is_running(process: subprocess.Popen, delay: float = 0.5) -> Tuple[bool, Optional[int], bytes, bytes]:
+    """Perform a quick check to make sure the browser started successfully."""
+    if not process:
+        return False, None, b"", b""
+        
+    # Check that process started without immediate termination
+    await asyncio.sleep(delay)
+    if process.poll() is not None:
+        # Process already terminated
+        stdout, stderr = b"", b""
+        try:
+            stdout, stderr = process.communicate(timeout=0.5)
+        except subprocess.TimeoutExpired:
+            pass
+
+        return False, process.returncode, stdout, stderr
+            
+
+    return True, 0, b"", b""
+
+
+def terminate_process(
+    pid: Union[int, str], 
+    timeout: float = 5.0,
+    force_kill_timeout: float = 3.0,
+    logger = None
+) -> Tuple[bool, Optional[str]]:
+    """
+    Robustly terminate a process across platforms with verification.
+    
+    Args:
+        pid: Process ID to terminate (int or string)
+        timeout: Seconds to wait for graceful termination before force killing
+        force_kill_timeout: Seconds to wait after force kill before considering it failed
+        logger: Optional logger object with error, warning, and info methods
+        
+    Returns:
+        Tuple of (success: bool, error_message: Optional[str])
+    """
+    # Convert pid to int if it's a string
+    if isinstance(pid, str):
+        try:
+            pid = int(pid)
+        except ValueError:
+            error_msg = f"Invalid PID format: {pid}"
+            if logger:
+                logger.error(error_msg)
+            return False, error_msg
+    
+    # Check if process exists
+    if not psutil.pid_exists(pid):
+        return True, None  # Process already terminated
+    
+    try:
+        process = psutil.Process(pid)
+        
+        # First attempt: graceful termination
+        if logger:
+            logger.info(f"Attempting graceful termination of process {pid}")
+            
+        if os.name == 'nt':  # Windows
+            subprocess.run(["taskkill", "/PID", str(pid)], 
+                          stdout=subprocess.DEVNULL, 
+                          stderr=subprocess.DEVNULL, 
+                          check=False)
+        else:  # Unix/Linux/MacOS
+            process.send_signal(signal.SIGTERM)
+        
+        # Wait for process to terminate
+        try:
+            process.wait(timeout=timeout)
+            if logger:
+                logger.info(f"Process {pid} terminated gracefully")
+            return True, None
+        except psutil.TimeoutExpired:
+            if logger:
+                logger.warning(f"Process {pid} did not terminate gracefully within {timeout} seconds, forcing termination")
+        
+        # Second attempt: force kill
+        if os.name == 'nt':  # Windows
+            subprocess.run(["taskkill", "/F", "/PID", str(pid)], 
+                          stdout=subprocess.DEVNULL, 
+                          stderr=subprocess.DEVNULL, 
+                          check=False)
+        else:  # Unix/Linux/MacOS
+            process.send_signal(signal.SIGKILL)
+        
+        # Verify process is killed
+        gone, alive = psutil.wait_procs([process], timeout=force_kill_timeout)
+        if process in alive:
+            error_msg = f"Failed to kill process {pid} even after force kill"
+            if logger:
+                logger.error(error_msg)
+            return False, error_msg
+            
+        if logger:
+            logger.info(f"Process {pid} terminated by force")
+        return True, None
+        
+    except psutil.NoSuchProcess:
+        # Process terminated while we were working with it
+        if logger:
+            logger.info(f"Process {pid} already terminated")
+        return True, None
+        
+    except Exception as e:
+        error_msg = f"Error terminating process {pid}: {str(e)}"
+        if logger:
+            logger.error(error_msg)
+        return False, error_msg

crawl4ai/browser_manager.py

@@ -0,0 +1,932 @@
+import asyncio
+import time
+from typing import List, Optional
+import os
+import sys
+import shutil
+import tempfile
+import subprocess
+from playwright.async_api import BrowserContext
+import hashlib
+from .js_snippet import load_js_script
+from .config import DOWNLOAD_PAGE_TIMEOUT
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from playwright_stealth import StealthConfig
+from .utils import get_chromium_path
+
+stealth_config = StealthConfig(
+    webdriver=True,
+    chrome_app=True,
+    chrome_csi=True,
+    chrome_load_times=True,
+    chrome_runtime=True,
+    navigator_languages=True,
+    navigator_plugins=True,
+    navigator_permissions=True,
+    webgl_vendor=True,
+    outerdimensions=True,
+    navigator_hardware_concurrency=True,
+    media_codecs=True,
+)
+
+BROWSER_DISABLE_OPTIONS = [
+    "--disable-background-networking",
+    "--disable-background-timer-throttling",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-breakpad",
+    "--disable-client-side-phishing-detection",
+    "--disable-component-extensions-with-background-pages",
+    "--disable-default-apps",
+    "--disable-extensions",
+    "--disable-features=TranslateUI",
+    "--disable-hang-monitor",
+    "--disable-ipc-flooding-protection",
+    "--disable-popup-blocking",
+    "--disable-prompt-on-repost",
+    "--disable-sync",
+    "--force-color-profile=srgb",
+    "--metrics-recording-only",
+    "--no-first-run",
+    "--password-store=basic",
+    "--use-mock-keychain",
+]
+
+
+class ManagedBrowser:
+    """
+    Manages the browser process and context. This class allows to connect to the browser using CDP protocol.
+
+    Attributes:
+        browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                            Default: "chromium".
+        user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                     temporary directory may be used. Default: None.
+        headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                         Default: True.
+        browser_process (subprocess.Popen): The process object for the browser.
+        temp_dir (str): Temporary directory for user data if not provided.
+        debugging_port (int): Port for debugging the browser.
+        host (str): Host for debugging the browser.
+
+        Methods:
+            start(): Starts the browser process and returns the CDP endpoint URL.
+            _get_browser_path(): Returns the browser executable path based on OS and browser type.
+            _get_browser_args(): Returns browser-specific command line arguments.
+            _get_user_data_dir(): Returns the user data directory path.
+            _cleanup(): Terminates the browser process and removes the temporary directory.
+            create_profile(): Static method to create a user profile by launching a browser for user interaction.
+    """
+
+    browser_type: str
+    user_data_dir: str
+    headless: bool
+    browser_process: subprocess.Popen
+    temp_dir: str
+    debugging_port: int
+    host: str
+
+    def __init__(
+        self,
+        browser_type: str = "chromium",
+        user_data_dir: Optional[str] = None,
+        headless: bool = False,
+        logger=None,
+        host: str = "localhost",
+        debugging_port: int = 9222,
+        cdp_url: Optional[str] = None, 
+    ):
+        """
+        Initialize the ManagedBrowser instance.
+
+        Args:
+            browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                                Default: "chromium".
+            user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                         temporary directory may be used. Default: None.
+            headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                             Default: True.
+            logger (logging.Logger): Logger instance for logging messages. Default: None.
+            host (str): Host for debugging the browser. Default: "localhost".
+            debugging_port (int): Port for debugging the browser. Default: 9222.
+            cdp_url (str or None): CDP URL to connect to the browser. Default: None.
+        """
+        self.browser_type = browser_type
+        self.user_data_dir = user_data_dir
+        self.headless = headless
+        self.browser_process = None
+        self.temp_dir = None
+        self.debugging_port = debugging_port
+        self.host = host
+        self.logger = logger
+        self.shutting_down = False
+        self.cdp_url = cdp_url
+
+    async def start(self) -> str:
+        """
+        Starts the browser process or returns CDP endpoint URL.
+        If cdp_url is provided, returns it directly.
+        If user_data_dir is not provided for local browser, creates a temporary directory.
+        
+        Returns:
+            str: CDP endpoint URL
+        """
+        # If CDP URL provided, just return it
+        if self.cdp_url:
+            return self.cdp_url
+
+        # Create temp dir if needed
+        if not self.user_data_dir:
+            self.temp_dir = tempfile.mkdtemp(prefix="browser-profile-")
+            self.user_data_dir = self.temp_dir
+
+        # Get browser path and args based on OS and browser type
+        # browser_path = self._get_browser_path()
+        args = await self._get_browser_args()
+
+        # Start browser process
+        try:
+            # Use DETACHED_PROCESS flag on Windows to fully detach the process
+            # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group
+            if sys.platform == "win32":
+                self.browser_process = subprocess.Popen(
+                    args, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
+                )
+            else:
+                self.browser_process = subprocess.Popen(
+                    args, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    preexec_fn=os.setpgrp  # Start in a new process group
+                )
+                
+            # We'll monitor for a short time to make sure it starts properly, but won't keep monitoring
+            await asyncio.sleep(0.5)  # Give browser time to start
+            await self._initial_startup_check()
+            await asyncio.sleep(2)  # Give browser time to start
+            return f"http://{self.host}:{self.debugging_port}"
+        except Exception as e:
+            await self.cleanup()
+            raise Exception(f"Failed to start browser: {e}")
+
+    async def _initial_startup_check(self):
+        """
+        Perform a quick check to make sure the browser started successfully.
+        This only runs once at startup rather than continuously monitoring.
+        """
+        if not self.browser_process:
+            return
+            
+        # Check that process started without immediate termination
+        await asyncio.sleep(0.5)
+        if self.browser_process.poll() is not None:
+            # Process already terminated
+            stdout, stderr = b"", b""
+            try:
+                stdout, stderr = self.browser_process.communicate(timeout=0.5)
+            except subprocess.TimeoutExpired:
+                pass
+                
+            self.logger.error(
+                message="Browser process terminated during startup | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
+                tag="ERROR",
+                params={
+                    "code": self.browser_process.returncode,
+                    "stdout": stdout.decode() if stdout else "",
+                    "stderr": stderr.decode() if stderr else "",
+                },
+            )
+    
+    async def _monitor_browser_process(self):
+        """
+        Monitor the browser process for unexpected termination.
+
+        How it works:
+        1. Read stdout and stderr from the browser process.
+        2. If the process has terminated, log the error message and terminate the browser.
+        3. If the shutting_down flag is set, log the normal termination message.
+        4. If any other error occurs, log the error message.
+
+        Note: This method should be called in a separate task to avoid blocking the main event loop.
+        This is DEPRECATED and should not be used for builtin browsers that need to outlive the Python process.
+        """
+        if self.browser_process:
+            try:
+                stdout, stderr = await asyncio.gather(
+                    asyncio.to_thread(self.browser_process.stdout.read),
+                    asyncio.to_thread(self.browser_process.stderr.read),
+                )
+
+                # Check shutting_down flag BEFORE logging anything
+                if self.browser_process.poll() is not None:
+                    if not self.shutting_down:
+                        self.logger.error(
+                            message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
+                            tag="ERROR",
+                            params={
+                                "code": self.browser_process.returncode,
+                                "stdout": stdout.decode(),
+                                "stderr": stderr.decode(),
+                            },
+                        )
+                        await self.cleanup()
+                    else:
+                        self.logger.info(
+                            message="Browser process terminated normally | Code: {code}",
+                            tag="INFO",
+                            params={"code": self.browser_process.returncode},
+                        )
+            except Exception as e:
+                if not self.shutting_down:
+                    self.logger.error(
+                        message="Error monitoring browser process: {error}",
+                        tag="ERROR",
+                        params={"error": str(e)},
+                    )
+
+    def _get_browser_path_WIP(self) -> str:
+        """Returns the browser executable path based on OS and browser type"""
+        if sys.platform == "darwin":  # macOS
+            paths = {
+                "chromium": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                "firefox": "/Applications/Firefox.app/Contents/MacOS/firefox",
+                "webkit": "/Applications/Safari.app/Contents/MacOS/Safari",
+            }
+        elif sys.platform == "win32":  # Windows
+            paths = {
+                "chromium": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+                "firefox": "C:\\Program Files\\Mozilla Firefox\\firefox.exe",
+                "webkit": None,  # WebKit not supported on Windows
+            }
+        else:  # Linux
+            paths = {
+                "chromium": "google-chrome",
+                "firefox": "firefox",
+                "webkit": None,  # WebKit not supported on Linux
+            }
+
+        return paths.get(self.browser_type)
+
+    async def _get_browser_path(self) -> str:
+        browser_path = await get_chromium_path(self.browser_type)
+        return browser_path
+
+    async def _get_browser_args(self) -> List[str]:
+        """Returns browser-specific command line arguments"""
+        base_args = [await self._get_browser_path()]
+
+        if self.browser_type == "chromium":
+            args = [
+                f"--remote-debugging-port={self.debugging_port}",
+                f"--user-data-dir={self.user_data_dir}",
+            ]
+            if self.headless:
+                args.append("--headless=new")
+        elif self.browser_type == "firefox":
+            args = [
+                "--remote-debugging-port",
+                str(self.debugging_port),
+                "--profile",
+                self.user_data_dir,
+            ]
+            if self.headless:
+                args.append("--headless")
+        else:
+            raise NotImplementedError(f"Browser type {self.browser_type} not supported")
+
+        return base_args + args
+
+    async def cleanup(self):
+        """Cleanup browser process and temporary directory"""
+        # Set shutting_down flag BEFORE any termination actions
+        self.shutting_down = True
+
+        if self.browser_process:
+            try:
+                # For builtin browsers that should persist, we should check if it's a detached process
+                # Only terminate if we have proper control over the process
+                if not self.browser_process.poll():
+                    # Process is still running
+                    self.browser_process.terminate()
+                    # Wait for process to end gracefully
+                    for _ in range(10):  # 10 attempts, 100ms each
+                        if self.browser_process.poll() is not None:
+                            break
+                        await asyncio.sleep(0.1)
+
+                    # Force kill if still running
+                    if self.browser_process.poll() is None:
+                        if sys.platform == "win32":
+                            # On Windows we might need taskkill for detached processes
+                            try:
+                                subprocess.run(["taskkill", "/F", "/PID", str(self.browser_process.pid)])
+                            except Exception:
+                                self.browser_process.kill()
+                        else:
+                            self.browser_process.kill()
+                        await asyncio.sleep(0.1)  # Brief wait for kill to take effect
+
+            except Exception as e:
+                self.logger.error(
+                    message="Error terminating browser: {error}",
+                    tag="ERROR", 
+                    params={"error": str(e)},
+                )
+
+        if self.temp_dir and os.path.exists(self.temp_dir):
+            try:
+                shutil.rmtree(self.temp_dir)
+            except Exception as e:
+                self.logger.error(
+                    message="Error removing temporary directory: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)},
+                )
+                
+    # These methods have been moved to BrowserProfiler class
+    @staticmethod
+    async def create_profile(browser_config=None, profile_name=None, logger=None):
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Creates a browser profile by launching a browser for interactive user setup
+        and waits until the user closes it. The profile is stored in a directory that
+        can be used later with BrowserConfig.user_data_dir.
+        
+        Please use BrowserProfiler.create_profile() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            profile_path = await profiler.create_profile(profile_name="my-login-profile")
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler(logger=logger)
+        return await profiler.create_profile(profile_name=profile_name, browser_config=browser_config)
+    
+    @staticmethod
+    def list_profiles():
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Lists all available browser profiles in the Crawl4AI profiles directory.
+        
+        Please use BrowserProfiler.list_profiles() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            profiles = profiler.list_profiles()
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler()
+        return profiler.list_profiles()
+        
+    @staticmethod
+    def delete_profile(profile_name_or_path):
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Delete a browser profile by name or path.
+        
+        Please use BrowserProfiler.delete_profile() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            success = profiler.delete_profile("my-profile")
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler()
+        return profiler.delete_profile(profile_name_or_path)
+
+
+
+
+class BrowserManager:
+    """
+    Manages the browser instance and context.
+
+    Attributes:
+        config (BrowserConfig): Configuration object containing all browser settings
+        logger: Logger instance for recording events and errors
+        browser (Browser): The browser instance
+        default_context (BrowserContext): The default browser context
+        managed_browser (ManagedBrowser): The managed browser instance
+        playwright (Playwright): The Playwright instance
+        sessions (dict): Dictionary to store session information
+        session_ttl (int): Session timeout in seconds
+    """
+
+    _playwright_instance = None
+    
+    @classmethod
+    async def get_playwright(cls):
+        from playwright.async_api import async_playwright
+        if cls._playwright_instance is None:
+            cls._playwright_instance = await async_playwright().start()
+        return cls._playwright_instance    
+
+    def __init__(self, browser_config: BrowserConfig, logger=None):
+        """
+        Initialize the BrowserManager with a browser configuration.
+
+        Args:
+            browser_config (BrowserConfig): Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+        """
+        self.config: BrowserConfig = browser_config
+        self.logger = logger
+
+        # Browser state
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+
+        # Session management
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+
+        # Keep track of contexts by a "config signature," so each unique config reuses a single context
+        self.contexts_by_config = {}
+        self._contexts_lock = asyncio.Lock() 
+
+        # Initialize ManagedBrowser if needed
+        if self.config.use_managed_browser:
+            self.managed_browser = ManagedBrowser(
+                browser_type=self.config.browser_type,
+                user_data_dir=self.config.user_data_dir,
+                headless=self.config.headless,
+                logger=self.logger,
+                debugging_port=self.config.debugging_port,
+                cdp_url=self.config.cdp_url,
+            )
+
+    async def start(self):
+        """
+        Start the browser instance and set up the default context.
+
+        How it works:
+        1. Check if Playwright is already initialized.
+        2. If not, initialize Playwright.
+        3. If managed browser is used, start it and connect to the CDP endpoint.
+        4. If managed browser is not used, launch the browser and set up the default context.
+
+        Note: This method should be called in a separate task to avoid blocking the main event loop.
+        """
+        self.playwright  = await self.get_playwright()
+        if self.playwright is None:
+            from playwright.async_api import async_playwright
+
+            self.playwright = await async_playwright().start()
+
+        if self.config.cdp_url or self.config.use_managed_browser:
+            self.config.use_managed_browser = True
+            cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url
+            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+            else:
+                self.default_context = await self.create_browser_context()
+            await self.setup_context(self.default_context)
+        else:
+            browser_args = self._build_browser_args()
+
+            # Launch appropriate browser type
+            if self.config.browser_type == "firefox":
+                self.browser = await self.playwright.firefox.launch(**browser_args)
+            elif self.config.browser_type == "webkit":
+                self.browser = await self.playwright.webkit.launch(**browser_args)
+            else:
+                self.browser = await self.playwright.chromium.launch(**browser_args)
+
+            self.default_context = self.browser
+
+
+    def _build_browser_args(self) -> dict:
+        """Build browser launch arguments from config."""
+        args = [
+            "--disable-gpu",
+            "--disable-gpu-compositing",
+            "--disable-software-rasterizer",
+            "--no-sandbox",
+            "--disable-dev-shm-usage",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--disable-infobars",
+            "--window-position=0,0",
+            "--ignore-certificate-errors",
+            "--ignore-certificate-errors-spki-list",
+            "--disable-blink-features=AutomationControlled",
+            "--window-position=400,0",
+            "--disable-renderer-backgrounding",
+            "--disable-ipc-flooding-protection",
+            "--force-color-profile=srgb",
+            "--mute-audio",
+            "--disable-background-timer-throttling",
+            # "--single-process",
+            f"--window-size={self.config.viewport_width},{self.config.viewport_height}",
+        ]
+
+        if self.config.light_mode:
+            args.extend(BROWSER_DISABLE_OPTIONS)
+
+        if self.config.text_mode:
+            args.extend(
+                [
+                    "--blink-settings=imagesEnabled=false",
+                    "--disable-remote-fonts",
+                    "--disable-images",
+                    "--disable-javascript",
+                    "--disable-software-rasterizer",
+                    "--disable-dev-shm-usage",
+                ]
+            )
+
+        if self.config.extra_args:
+            args.extend(self.config.extra_args)
+
+        browser_args = {"headless": self.config.headless, "args": args}
+
+        if self.config.chrome_channel:
+            browser_args["channel"] = self.config.chrome_channel
+
+        if self.config.accept_downloads:
+            browser_args["downloads_path"] = self.config.downloads_path or os.path.join(
+                os.getcwd(), "downloads"
+            )
+            os.makedirs(browser_args["downloads_path"], exist_ok=True)
+
+        if self.config.proxy or self.config.proxy_config:
+            from playwright.async_api import ProxySettings
+
+            proxy_settings = (
+                ProxySettings(server=self.config.proxy)
+                if self.config.proxy
+                else ProxySettings(
+                    server=self.config.proxy_config.server,
+                    username=self.config.proxy_config.username,
+                    password=self.config.proxy_config.password,
+                )
+            )
+            browser_args["proxy"] = proxy_settings
+
+        return browser_args
+
+    async def setup_context(
+        self,
+        context: BrowserContext,
+        crawlerRunConfig: CrawlerRunConfig = None,
+        is_default=False,
+    ):
+        """
+        Set up a browser context with the configured options.
+
+        How it works:
+        1. Set extra HTTP headers if provided.
+        2. Add cookies if provided.
+        3. Load storage state if provided.
+        4. Accept downloads if enabled.
+        5. Set default timeouts for navigation and download.
+        6. Set user agent if provided.
+        7. Set browser hints if provided.
+        8. Set proxy if provided.
+        9. Set downloads path if provided.
+        10. Set storage state if provided.
+        11. Set cache if provided.
+        12. Set extra HTTP headers if provided.
+        13. Add cookies if provided.
+        14. Set default timeouts for navigation and download if enabled.
+        15. Set user agent if provided.
+        16. Set browser hints if provided.
+
+        Args:
+            context (BrowserContext): The browser context to set up
+            crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings
+            is_default (bool): Flag indicating if this is the default context
+        Returns:
+            None
+        """
+        if self.config.headers:
+            await context.set_extra_http_headers(self.config.headers)
+
+        if self.config.cookies:
+            await context.add_cookies(self.config.cookies)
+
+        if self.config.storage_state:
+            await context.storage_state(path=None)
+
+        if self.config.accept_downloads:
+            context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            if self.config.downloads_path:
+                context._impl_obj._options["accept_downloads"] = True
+                context._impl_obj._options[
+                    "downloads_path"
+                ] = self.config.downloads_path
+
+        # Handle user agent and browser hints
+        if self.config.user_agent:
+            combined_headers = {
+                "User-Agent": self.config.user_agent,
+                "sec-ch-ua": self.config.browser_hint,
+            }
+            combined_headers.update(self.config.headers)
+            await context.set_extra_http_headers(combined_headers)
+
+        # Add default cookie
+        await context.add_cookies(
+            [
+                {
+                    "name": "cookiesEnabled",
+                    "value": "true",
+                    "url": crawlerRunConfig.url
+                    if crawlerRunConfig
+                    else "https://crawl4ai.com/",
+                }
+            ]
+        )
+
+        # Handle navigator overrides
+        if crawlerRunConfig:
+            if (
+                crawlerRunConfig.override_navigator
+                or crawlerRunConfig.simulate_user
+                or crawlerRunConfig.magic
+            ):
+                await context.add_init_script(load_js_script("navigator_overrider"))        
+
+    async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None):
+        """
+        Creates and returns a new browser context with configured settings.
+        Applies text-only mode settings if text_mode is enabled in config.
+
+        Returns:
+            Context: Browser context object with the specified configurations
+        """
+        # Base settings
+        user_agent = self.config.headers.get("User-Agent", self.config.user_agent) 
+        viewport_settings = {
+            "width": self.config.viewport_width,
+            "height": self.config.viewport_height,
+        }
+        proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
+
+        blocked_extensions = [
+            # Images
+            "jpg",
+            "jpeg",
+            "png",
+            "gif",
+            "webp",
+            "svg",
+            "ico",
+            "bmp",
+            "tiff",
+            "psd",
+            # Fonts
+            "woff",
+            "woff2",
+            "ttf",
+            "otf",
+            "eot",
+            # Styles
+            # 'css', 'less', 'scss', 'sass',
+            # Media
+            "mp4",
+            "webm",
+            "ogg",
+            "avi",
+            "mov",
+            "wmv",
+            "flv",
+            "m4v",
+            "mp3",
+            "wav",
+            "aac",
+            "m4a",
+            "opus",
+            "flac",
+            # Documents
+            "pdf",
+            "doc",
+            "docx",
+            "xls",
+            "xlsx",
+            "ppt",
+            "pptx",
+            # Archives
+            "zip",
+            "rar",
+            "7z",
+            "tar",
+            "gz",
+            # Scripts and data
+            "xml",
+            "swf",
+            "wasm",
+        ]
+
+        # Common context settings
+        context_settings = {
+            "user_agent": user_agent,
+            "viewport": viewport_settings,
+            "proxy": proxy_settings,
+            "accept_downloads": self.config.accept_downloads,
+            "storage_state": self.config.storage_state,
+            "ignore_https_errors": self.config.ignore_https_errors,
+            "device_scale_factor": 1.0,
+            "java_script_enabled": self.config.java_script_enabled,
+        }
+        
+        if crawlerRunConfig:
+            # Check if there is value for crawlerRunConfig.proxy_config set add that to context
+            if crawlerRunConfig.proxy_config:
+                proxy_settings = {
+                    "server": crawlerRunConfig.proxy_config.server,
+                }
+                if crawlerRunConfig.proxy_config.username:
+                    proxy_settings.update({
+                        "username": crawlerRunConfig.proxy_config.username,
+                        "password": crawlerRunConfig.proxy_config.password,
+                    })
+                context_settings["proxy"] = proxy_settings
+
+        if self.config.text_mode:
+            text_mode_settings = {
+                "has_touch": False,
+                "is_mobile": False,
+            }
+            # Update context settings with text mode settings
+            context_settings.update(text_mode_settings)
+
+        # Create and return the context with all settings
+        context = await self.browser.new_context(**context_settings)
+
+        # Apply text mode settings if enabled
+        if self.config.text_mode:
+            # Create and apply route patterns for each extension
+            for ext in blocked_extensions:
+                await context.route(f"**/*.{ext}", lambda route: route.abort())
+        return context
+
+    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
+        """
+        Converts the crawlerRunConfig into a dict, excludes ephemeral fields,
+        then returns a hash of the sorted JSON. This yields a stable signature
+        that identifies configurations requiring a unique browser context.
+        """
+        import json
+
+        config_dict = crawlerRunConfig.__dict__.copy()
+        # Exclude items that do not affect browser-level setup.
+        # Expand or adjust as needed, e.g. chunking_strategy is purely for data extraction, not for browser config.
+        ephemeral_keys = [
+            "session_id",
+            "js_code",
+            "scraping_strategy",
+            "extraction_strategy",
+            "chunking_strategy",
+            "cache_mode",
+            "content_filter",
+            "semaphore_count",
+            "url"
+        ]
+        for key in ephemeral_keys:
+            if key in config_dict:
+                del config_dict[key]
+        # Convert to canonical JSON string
+        signature_json = json.dumps(config_dict, sort_keys=True, default=str)
+
+        # Hash the JSON so we get a compact, unique string
+        signature_hash = hashlib.sha256(signature_json.encode("utf-8")).hexdigest()
+        return signature_hash
+
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig):
+        """
+        Get a page for the given session ID, creating a new one if needed.
+
+        Args:
+            crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings
+
+        Returns:
+            (page, context): The Page and its BrowserContext
+        """
+        self._cleanup_expired_sessions()
+
+        # If a session_id is provided and we already have it, reuse that page + context
+        if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions:
+            context, page, _ = self.sessions[crawlerRunConfig.session_id]
+            # Update last-used timestamp
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            return page, context
+
+        # If using a managed browser, just grab the shared default_context
+        if self.config.use_managed_browser:
+            context = self.default_context
+            pages = context.pages
+            page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+            if not page:
+                page = await context.new_page()
+        else:
+            # Otherwise, check if we have an existing context for this config
+            config_signature = self._make_config_signature(crawlerRunConfig)
+
+            async with self._contexts_lock:
+                if config_signature in self.contexts_by_config:
+                    context = self.contexts_by_config[config_signature]
+                else:
+                    # Create and setup a new context
+                    context = await self.create_browser_context(crawlerRunConfig)
+                    await self.setup_context(context, crawlerRunConfig)
+                    self.contexts_by_config[config_signature] = context
+
+            # Create a new page from the chosen context
+            page = await context.new_page()
+
+        # If a session_id is specified, store this session so we can reuse later
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+
+        return page, context
+
+    async def kill_session(self, session_id: str):
+        """
+        Kill a browser session and clean up resources.
+
+        Args:
+            session_id (str): The session ID to kill.
+        """
+        if session_id in self.sessions:
+            context, page, _ = self.sessions[session_id]
+            await page.close()
+            if not self.config.use_managed_browser:
+                await context.close()
+            del self.sessions[session_id]
+
+    def _cleanup_expired_sessions(self):
+        """Clean up expired sessions based on TTL."""
+        current_time = time.time()
+        expired_sessions = [
+            sid
+            for sid, (_, _, last_used) in self.sessions.items()
+            if current_time - last_used > self.session_ttl
+        ]
+        for sid in expired_sessions:
+            asyncio.create_task(self.kill_session(sid))
+
+    async def close(self):
+        """Close all browser resources and clean up."""
+        if self.config.cdp_url:
+            return
+        
+        if self.config.sleep_on_close:
+            await asyncio.sleep(0.5)
+
+        session_ids = list(self.sessions.keys())
+        for session_id in session_ids:
+            await self.kill_session(session_id)
+
+        # Now close all contexts we created. This reclaims memory from ephemeral contexts.
+        for ctx in self.contexts_by_config.values():
+            try:
+                await ctx.close()
+            except Exception as e:
+                self.logger.error(
+                    message="Error closing context: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)}
+                )
+        self.contexts_by_config.clear()
+
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+
+        if self.managed_browser:
+            await asyncio.sleep(0.5)
+            await self.managed_browser.cleanup()
+            self.managed_browser = None
+
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None

crawl4ai/browser_profiler.py

@@ -0,0 +1,974 @@
+"""
+Browser Profiler Module
+
+This module provides a dedicated class for managing browser profiles
+that can be used for identity-based crawling with Crawl4AI.
+"""
+
+import os
+import asyncio
+import signal
+import sys
+import datetime
+import uuid
+import shutil
+import json
+import subprocess
+import time
+from typing import List, Dict, Optional, Any, Tuple
+from colorama import Fore, Style, init
+
+from .async_configs import BrowserConfig
+from .browser_manager import ManagedBrowser
+from .async_logger import AsyncLogger, AsyncLoggerBase
+from .utils import get_home_folder
+
+
+class BrowserProfiler:
+    """
+    A dedicated class for managing browser profiles for Crawl4AI.
+    
+    The BrowserProfiler allows you to:
+    - Create browser profiles interactively
+    - List available profiles
+    - Delete profiles when no longer needed
+    - Get profile paths for use in BrowserConfig
+    
+    Profiles are stored by default in ~/.crawl4ai/profiles/
+    """
+    
+    def __init__(self, logger: Optional[AsyncLoggerBase] = None):
+        """
+        Initialize the BrowserProfiler.
+        
+        Args:
+            logger (AsyncLoggerBase, optional): Logger for outputting messages.
+                If None, a default AsyncLogger will be created.
+        """
+        # Initialize colorama for colorful terminal output
+        init()
+        
+        # Create a logger if not provided
+        if logger is None:
+            self.logger = AsyncLogger(verbose=True)
+        elif not isinstance(logger, AsyncLoggerBase):
+            self.logger = AsyncLogger(verbose=True)
+        else:
+            self.logger = logger
+            
+        # Ensure profiles directory exists
+        self.profiles_dir = os.path.join(get_home_folder(), "profiles")
+        os.makedirs(self.profiles_dir, exist_ok=True)
+        
+        # Builtin browser config file
+        self.builtin_browser_dir = os.path.join(get_home_folder(), "builtin-browser")
+        self.builtin_config_file = os.path.join(self.builtin_browser_dir, "browser_config.json")
+        os.makedirs(self.builtin_browser_dir, exist_ok=True)
+    
+    async def create_profile(self, 
+                            profile_name: Optional[str] = None, 
+                            browser_config: Optional[BrowserConfig] = None) -> Optional[str]:
+        """
+        Creates a browser profile by launching a browser for interactive user setup
+        and waits until the user closes it. The profile is stored in a directory that
+        can be used later with BrowserConfig.user_data_dir.
+        
+        Args:
+            profile_name (str, optional): Name for the profile directory.
+                If None, a name is generated based on timestamp.
+            browser_config (BrowserConfig, optional): Configuration for the browser.
+                If None, a default configuration is used with headless=False.
+                
+        Returns:
+            str: Path to the created profile directory, or None if creation failed
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Create a profile interactively
+            profile_path = await profiler.create_profile(
+                profile_name="my-login-profile"
+            )
+            
+            # Use the profile in a crawler
+            browser_config = BrowserConfig(
+                headless=True,
+                use_managed_browser=True,
+                user_data_dir=profile_path
+            )
+            
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                # The crawler will now use your profile with all your cookies and login state
+                result = await crawler.arun("https://example.com/dashboard")
+            ```
+        """
+        # Create default browser config if none provided
+        if browser_config is None:
+            from .async_configs import BrowserConfig
+            browser_config = BrowserConfig(
+                browser_type="chromium",
+                headless=False,  # Must be visible for user interaction
+                verbose=True
+            )
+        else:
+            # Ensure headless is False for user interaction
+            browser_config.headless = False
+            
+        # Generate profile name if not provided
+        if not profile_name:
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"profile_{timestamp}_{uuid.uuid4().hex[:6]}"
+            
+        # Sanitize profile name (replace spaces and special chars)
+        profile_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in profile_name)
+        
+        # Set user data directory
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        os.makedirs(profile_path, exist_ok=True)
+        
+        # Print instructions for the user with colorama formatting
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="PROFILE")
+        self.logger.info(f"Creating browser profile: {Fore.GREEN}{profile_name}{Style.RESET_ALL}", tag="PROFILE")
+        self.logger.info(f"Profile directory: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+        
+        self.logger.info("\nInstructions:", tag="PROFILE")
+        self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE")
+        self.logger.info(f"2. {Fore.CYAN}Log in to websites{Style.RESET_ALL}, configure settings, etc. as needed.", tag="PROFILE")
+        self.logger.info(f"3. When you're done, {Fore.YELLOW}press 'q' in this terminal{Style.RESET_ALL} to close the browser.", tag="PROFILE")
+        self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE")
+        self.logger.info(f"{border}\n", tag="PROFILE")
+        
+        # Create managed browser instance
+        managed_browser = ManagedBrowser(
+            browser_type=browser_config.browser_type,
+            user_data_dir=profile_path,
+            headless=False,  # Must be visible
+            logger=self.logger,
+            debugging_port=browser_config.debugging_port
+        )
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="PROFILE")
+            await managed_browser.cleanup()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Profile creation interrupted. Profile may be incomplete.", tag="PROFILE")
+                sys.exit(1)
+                
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user is done with the browser
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' when you've finished using the browser...{Style.RESET_ALL}", tag="PROFILE")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser and saving profile...{Style.RESET_ALL}", tag="PROFILE")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if managed_browser.browser_process and managed_browser.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="PROFILE")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+        
+        try:
+            # Start the browser
+            await managed_browser.start()
+            
+            # Check if browser started successfully
+            browser_process = managed_browser.browser_process
+            if not browser_process:
+                self.logger.error("Failed to start browser process.", tag="PROFILE")
+                return None
+            
+            self.logger.info(f"Browser launched. {Fore.CYAN}Waiting for you to finish...{Style.RESET_ALL}", tag="PROFILE") 
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for either the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="PROFILE")
+                await managed_browser.cleanup()
+            
+            self.logger.success(f"Browser closed. Profile saved at: {Fore.GREEN}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+                
+        except Exception as e:
+            self.logger.error(f"Error creating profile: {str(e)}", tag="PROFILE")
+            await managed_browser.cleanup()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await managed_browser.cleanup()
+        
+        # Return the profile path
+        return profile_path
+    
+    def list_profiles(self) -> List[Dict[str, Any]]:
+        """
+        Lists all available browser profiles in the Crawl4AI profiles directory.
+        
+        Returns:
+            list: A list of dictionaries containing profile information:
+                  [{"name": "profile_name", "path": "/path/to/profile", "created": datetime, "type": "chromium|firefox"}]
+                  
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # List all available profiles
+            profiles = profiler.list_profiles()
+            
+            for profile in profiles:
+                print(f"Profile: {profile['name']}")
+                print(f"  Path: {profile['path']}")
+                print(f"  Created: {profile['created']}")
+                print(f"  Browser type: {profile['type']}")
+            ```
+        """
+        if not os.path.exists(self.profiles_dir):
+            return []
+            
+        profiles = []
+        
+        for name in os.listdir(self.profiles_dir):
+            profile_path = os.path.join(self.profiles_dir, name)
+            
+            # Skip if not a directory
+            if not os.path.isdir(profile_path):
+                continue
+                
+            # Check if this looks like a valid browser profile
+            # For Chromium: Look for Preferences file
+            # For Firefox: Look for prefs.js file
+            is_valid = False
+            
+            if os.path.exists(os.path.join(profile_path, "Preferences")) or \
+               os.path.exists(os.path.join(profile_path, "Default", "Preferences")):
+                is_valid = "chromium"
+            elif os.path.exists(os.path.join(profile_path, "prefs.js")):
+                is_valid = "firefox"
+                
+            if is_valid:
+                # Get creation time
+                created = datetime.datetime.fromtimestamp(
+                    os.path.getctime(profile_path)
+                )
+                
+                profiles.append({
+                    "name": name,
+                    "path": profile_path,
+                    "created": created,
+                    "type": is_valid
+                })
+                
+        # Sort by creation time, newest first
+        profiles.sort(key=lambda x: x["created"], reverse=True)
+        
+        return profiles
+    
+    def get_profile_path(self, profile_name: str) -> Optional[str]:
+        """
+        Get the full path to a profile by name.
+        
+        Args:
+            profile_name (str): Name of the profile (not the full path)
+            
+        Returns:
+            str: Full path to the profile directory, or None if not found
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            path = profiler.get_profile_path("my-profile")
+            if path:
+                print(f"Profile path: {path}")
+            else:
+                print("Profile not found")
+            ```
+        """
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            # Chrck if profile_name itself is full path
+            if os.path.isabs(profile_name):
+                profile_path = profile_name
+            else:
+                return None
+        
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return None  # Not a valid browser profile
+            
+        return profile_path
+    
+    def delete_profile(self, profile_name_or_path: str) -> bool:
+        """
+        Delete a browser profile by name or path.
+        
+        Args:
+            profile_name_or_path (str): Name of the profile or full path to profile directory
+            
+        Returns:
+            bool: True if the profile was deleted successfully, False otherwise
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Delete by name
+            success = profiler.delete_profile("my-profile")
+            
+            # Delete by path
+            success = profiler.delete_profile("/path/to/.crawl4ai/profiles/my-profile")
+            ```
+        """
+        # Determine if input is a name or a path
+        if os.path.isabs(profile_name_or_path):
+            # Full path provided
+            profile_path = profile_name_or_path
+        else:
+            # Just a name provided, construct path
+            profile_path = os.path.join(self.profiles_dir, profile_name_or_path)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            return False
+            
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return False  # Not a valid browser profile
+            
+        # Delete the profile directory
+        try:
+            shutil.rmtree(profile_path)
+            return True
+        except Exception:
+            return False
+            
+    async def interactive_manager(self, crawl_callback=None):
+        """
+        Launch an interactive profile management console.
+        
+        Args:
+            crawl_callback (callable, optional): Function to call when selecting option to use 
+                a profile for crawling. It will be called with (profile_path, url).
+                
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Define a custom crawl function
+            async def my_crawl_function(profile_path, url):
+                print(f"Crawling {url} with profile {profile_path}")
+                # Implement your crawling logic here
+                
+            # Start interactive manager
+            await profiler.interactive_manager(crawl_callback=my_crawl_function)
+            ```
+        """
+        while True:
+            self.logger.info(f"\n{Fore.CYAN}Profile Management Options:{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"1. {Fore.GREEN}Create a new profile{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"2. {Fore.YELLOW}List available profiles{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"3. {Fore.RED}Delete a profile{Style.RESET_ALL}", tag="MENU")
+            
+            # Only show crawl option if callback provided
+            if crawl_callback:
+                self.logger.info(f"4. {Fore.CYAN}Use a profile to crawl a website{Style.RESET_ALL}", tag="MENU")
+                self.logger.info(f"5. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "5"
+            else:
+                self.logger.info(f"4. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "4"
+            
+            choice = input(f"\n{Fore.CYAN}Enter your choice (1-{exit_option}): {Style.RESET_ALL}")
+            
+            if choice == "1":
+                # Create new profile
+                name = input(f"{Fore.GREEN}Enter a name for the new profile (or press Enter for auto-generated name): {Style.RESET_ALL}")
+                await self.create_profile(name or None)
+                
+            elif choice == "2":
+                # List profiles
+                profiles = self.list_profiles()
+                
+                if not profiles:
+                    self.logger.warning("  No profiles found. Create one first with option 1.", tag="PROFILES")
+                    continue
+                
+                # Print profile information with colorama formatting
+                self.logger.info("\nAvailable profiles:", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {Fore.CYAN}{profile['name']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Path: {Fore.YELLOW}{profile['path']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES")
+                    self.logger.info(f"    Browser type: {profile['type']}", tag="PROFILES")
+                    self.logger.info("", tag="PROFILES")  # Empty line for spacing
+                
+            elif choice == "3":
+                # Delete profile
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found to delete", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to delete
+                profile_idx = input(f"{Fore.RED}Enter the number of the profile to delete (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_name = profiles[idx]["name"]
+                        self.logger.info(f"Deleting profile: {Fore.YELLOW}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                        
+                        # Confirm deletion
+                        confirm = input(f"{Fore.RED}Are you sure you want to delete this profile? (y/n): {Style.RESET_ALL}")
+                        if confirm.lower() == 'y':
+                            success = self.delete_profile(profiles[idx]["path"])
+                            
+                            if success:
+                                self.logger.success(f"Profile {Fore.GREEN}{profile_name}{Style.RESET_ALL} deleted successfully", tag="PROFILES")
+                            else:
+                                self.logger.error(f"Failed to delete profile {Fore.RED}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == "4" and crawl_callback:
+                # Use profile to crawl a site
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found. Create one first.", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to use
+                profile_idx = input(f"{Fore.CYAN}Enter the number of the profile to use (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_path = profiles[idx]["path"]
+                        url = input(f"{Fore.CYAN}Enter the URL to crawl: {Style.RESET_ALL}")
+                        if url:
+                            # Call the provided crawl callback
+                            await crawl_callback(profile_path, url)
+                        else:
+                            self.logger.error("No URL provided", tag="CRAWL")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == exit_option:
+                # Exit
+                self.logger.info("Exiting profile management", tag="MENU")
+                break
+                
+            else:
+                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")
+
+    async def launch_standalone_browser(self, 
+                                  browser_type: str = "chromium",
+                                  user_data_dir: Optional[str] = None,
+                                  debugging_port: int = 9222,
+                                  headless: bool = False,
+                                  save_as_builtin: bool = False) -> Optional[str]:
+        """
+        Launch a standalone browser with CDP debugging enabled and keep it running
+        until the user presses 'q'. Returns and displays the CDP URL.
+        
+        Args:
+            browser_type (str): Type of browser to launch ('chromium' or 'firefox')
+            user_data_dir (str, optional): Path to user profile directory
+            debugging_port (int): Port to use for CDP debugging
+            headless (bool): Whether to run in headless mode
+            
+        Returns:
+            str: CDP URL for the browser, or None if launch failed
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            cdp_url = await profiler.launch_standalone_browser(
+                user_data_dir="/path/to/profile",
+                debugging_port=9222
+            )
+            # Use cdp_url to connect to the browser
+            ```
+        """
+        # Use the provided directory if specified, otherwise create a temporary directory
+        if user_data_dir:
+            # Directory is provided directly, ensure it exists
+            profile_path = user_data_dir
+            os.makedirs(profile_path, exist_ok=True)
+        else:
+            # Create a temporary profile directory
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"temp_{timestamp}_{uuid.uuid4().hex[:6]}"
+            profile_path = os.path.join(self.profiles_dir, profile_name)
+            os.makedirs(profile_path, exist_ok=True)
+        
+        # Print initial information
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="CDP")
+        self.logger.info(f"Launching standalone browser with CDP debugging", tag="CDP")
+        self.logger.info(f"Browser type: {Fore.GREEN}{browser_type}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Profile path: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Debugging port: {Fore.CYAN}{debugging_port}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Headless mode: {Fore.CYAN}{headless}{Style.RESET_ALL}", tag="CDP")
+        
+        # Create managed browser instance
+        managed_browser = ManagedBrowser(
+            browser_type=browser_type,
+            user_data_dir=profile_path,
+            headless=headless,
+            logger=self.logger,
+            debugging_port=debugging_port
+        )
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="CDP")
+            await managed_browser.cleanup()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Browser terminated by user.", tag="CDP")
+                sys.exit(1)
+                    
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user wants to exit
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' to stop the browser and exit...{Style.RESET_ALL}", tag="CDP")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser...{Style.RESET_ALL}", tag="CDP")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if managed_browser.browser_process and managed_browser.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="CDP")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+                
+        # Function to retrieve and display CDP JSON config
+        async def get_cdp_json(port):
+            import aiohttp
+            cdp_url = f"http://localhost:{port}"
+            json_url = f"{cdp_url}/json/version"
+            
+            try:
+                async with aiohttp.ClientSession() as session:
+                    # Try multiple times in case the browser is still starting up
+                    for _ in range(10):
+                        try:
+                            async with session.get(json_url) as response:
+                                if response.status == 200:
+                                    data = await response.json()
+                                    return cdp_url, data
+                        except Exception:
+                            pass
+                        
+                        await asyncio.sleep(0.5)
+                    
+                    return cdp_url, None
+            except Exception as e:
+                self.logger.error(f"Error fetching CDP JSON: {str(e)}", tag="CDP")
+                return cdp_url, None
+        
+        cdp_url = None
+        config_json = None
+        
+        try:
+            # Start the browser
+            await managed_browser.start()
+            
+            # Check if browser started successfully
+            browser_process = managed_browser.browser_process
+            if not browser_process:
+                self.logger.error("Failed to start browser process.", tag="CDP")
+                return None
+            
+            self.logger.info(f"Browser launched successfully. Retrieving CDP information...", tag="CDP") 
+            
+            # Get CDP URL and JSON config
+            cdp_url, config_json = await get_cdp_json(debugging_port)
+            
+            if cdp_url:
+                self.logger.success(f"CDP URL: {Fore.GREEN}{cdp_url}{Style.RESET_ALL}", tag="CDP")
+                
+                if config_json:
+                    # Display relevant CDP information
+                    self.logger.info(f"Browser: {Fore.CYAN}{config_json.get('Browser', 'Unknown')}{Style.RESET_ALL}", tag="CDP")
+                    self.logger.info(f"Protocol Version: {config_json.get('Protocol-Version', 'Unknown')}", tag="CDP")
+                    if 'webSocketDebuggerUrl' in config_json:
+                        self.logger.info(f"WebSocket URL: {Fore.GREEN}{config_json['webSocketDebuggerUrl']}{Style.RESET_ALL}", tag="CDP")
+                else:
+                    self.logger.warning("Could not retrieve CDP configuration JSON", tag="CDP")
+            else:
+                self.logger.error(f"Failed to get CDP URL on port {debugging_port}", tag="CDP")
+                await managed_browser.cleanup()
+                return None
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="CDP")
+                await managed_browser.cleanup()
+            
+            self.logger.success(f"Browser closed.", tag="CDP")
+                
+        except Exception as e:
+            self.logger.error(f"Error launching standalone browser: {str(e)}", tag="CDP")
+            await managed_browser.cleanup()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await managed_browser.cleanup()
+        
+        # Return the CDP URL
+        return cdp_url
+    
+    async def launch_builtin_browser(self, 
+                                 browser_type: str = "chromium",
+                                 debugging_port: int = 9222,
+                                 headless: bool = True) -> Optional[str]:
+        """
+        Launch a browser in the background for use as the builtin browser.
+        
+        Args:
+            browser_type (str): Type of browser to launch ('chromium' or 'firefox')
+            debugging_port (int): Port to use for CDP debugging
+            headless (bool): Whether to run in headless mode
+            
+        Returns:
+            str: CDP URL for the browser, or None if launch failed
+        """
+        # Check if there's an existing browser still running
+        browser_info = self.get_builtin_browser_info()
+        if browser_info and self._is_browser_running(browser_info.get('pid')):
+            self.logger.info("Builtin browser is already running", tag="BUILTIN")
+            return browser_info.get('cdp_url')
+        
+        # Create a user data directory for the builtin browser
+        user_data_dir = os.path.join(self.builtin_browser_dir, "user_data")
+        os.makedirs(user_data_dir, exist_ok=True)
+        
+        # Create managed browser instance
+        managed_browser = ManagedBrowser(
+            browser_type=browser_type,
+            user_data_dir=user_data_dir,
+            headless=headless,
+            logger=self.logger,
+            debugging_port=debugging_port
+        )
+        
+        try:
+            # Start the browser
+            await managed_browser.start()
+            
+            # Check if browser started successfully
+            browser_process = managed_browser.browser_process
+            if not browser_process:
+                self.logger.error("Failed to start browser process.", tag="BUILTIN")
+                return None
+            
+            # Get CDP URL
+            cdp_url = f"http://localhost:{debugging_port}"
+            
+            # Try to verify browser is responsive by fetching version info
+            import aiohttp
+            json_url = f"{cdp_url}/json/version"
+            config_json = None
+            
+            try:
+                async with aiohttp.ClientSession() as session:
+                    for _ in range(10):  # Try multiple times
+                        try:
+                            async with session.get(json_url) as response:
+                                if response.status == 200:
+                                    config_json = await response.json()
+                                    break
+                        except Exception:
+                            pass
+                        await asyncio.sleep(0.5)
+            except Exception as e:
+                self.logger.warning(f"Could not verify browser: {str(e)}", tag="BUILTIN")
+            
+            # Save browser info
+            browser_info = {
+                'pid': browser_process.pid,
+                'cdp_url': cdp_url,
+                'user_data_dir': user_data_dir,
+                'browser_type': browser_type,
+                'debugging_port': debugging_port,
+                'start_time': time.time(),
+                'config': config_json
+            }
+            
+            with open(self.builtin_config_file, 'w') as f:
+                json.dump(browser_info, f, indent=2)
+                
+            # Detach from the browser process - don't keep any references
+            # This is important to allow the Python script to exit while the browser continues running
+            # We'll just record the PID and other info, and the browser will run independently
+            managed_browser.browser_process = None
+                
+            self.logger.success(f"Builtin browser launched at CDP URL: {cdp_url}", tag="BUILTIN")
+            return cdp_url
+            
+        except Exception as e:
+            self.logger.error(f"Error launching builtin browser: {str(e)}", tag="BUILTIN")
+            if managed_browser:
+                await managed_browser.cleanup()
+            return None
+    
+    def get_builtin_browser_info(self) -> Optional[Dict[str, Any]]:
+        """
+        Get information about the builtin browser.
+        
+        Returns:
+            dict: Browser information or None if no builtin browser is configured
+        """
+        if not os.path.exists(self.builtin_config_file):
+            return None
+            
+        try:
+            with open(self.builtin_config_file, 'r') as f:
+                browser_info = json.load(f)
+                
+            # Check if the browser is still running
+            if not self._is_browser_running(browser_info.get('pid')):
+                self.logger.warning("Builtin browser is not running", tag="BUILTIN")
+                return None
+                
+            return browser_info
+        except Exception as e:
+            self.logger.error(f"Error reading builtin browser config: {str(e)}", tag="BUILTIN")
+            return None
+            
+    def _is_browser_running(self, pid: Optional[int]) -> bool:
+        """Check if a process with the given PID is running"""
+        if not pid:
+            return False
+            
+        try:
+            # Check if the process exists
+            if sys.platform == "win32":
+                process = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], 
+                                         capture_output=True, text=True)
+                return str(pid) in process.stdout
+            else:
+                # Unix-like systems
+                os.kill(pid, 0)  # This doesn't actually kill the process, just checks if it exists
+            return True
+        except (ProcessLookupError, PermissionError, OSError):
+            return False
+            
+    async def kill_builtin_browser(self) -> bool:
+        """
+        Kill the builtin browser if it's running.
+        
+        Returns:
+            bool: True if the browser was killed, False otherwise
+        """
+        browser_info = self.get_builtin_browser_info()
+        if not browser_info:
+            self.logger.warning("No builtin browser found", tag="BUILTIN")
+            return False
+            
+        pid = browser_info.get('pid')
+        if not pid:
+            return False
+            
+        try:
+            if sys.platform == "win32":
+                subprocess.run(["taskkill", "/F", "/PID", str(pid)], check=True)
+            else:
+                os.kill(pid, signal.SIGTERM)
+                # Wait for termination
+                for _ in range(5):
+                    if not self._is_browser_running(pid):
+                        break
+                    await asyncio.sleep(0.5)
+                else:
+                    # Force kill if still running
+                    os.kill(pid, signal.SIGKILL)
+                    
+            # Remove config file
+            if os.path.exists(self.builtin_config_file):
+                os.unlink(self.builtin_config_file)
+                
+            self.logger.success("Builtin browser terminated", tag="BUILTIN")
+            return True
+        except Exception as e:
+            self.logger.error(f"Error killing builtin browser: {str(e)}", tag="BUILTIN")
+            return False
+    
+    async def get_builtin_browser_status(self) -> Dict[str, Any]:
+        """
+        Get status information about the builtin browser.
+        
+        Returns:
+            dict: Status information with running, cdp_url, and info fields
+        """
+        browser_info = self.get_builtin_browser_info()
+        
+        if not browser_info:
+            return {
+                'running': False,
+                'cdp_url': None,
+                'info': None
+            }
+            
+        return {
+            'running': True,
+            'cdp_url': browser_info.get('cdp_url'),
+            'info': browser_info
+        }
+

crawl4ai/cache_context.py

@@ -0,0 +1,117 @@
+from enum import Enum
+
+
+class CacheMode(Enum):
+    """
+    Defines the caching behavior for web crawling operations.
+
+    Modes:
+    - ENABLED: Normal caching behavior (read and write)
+    - DISABLED: No caching at all
+    - READ_ONLY: Only read from cache, don't write
+    - WRITE_ONLY: Only write to cache, don't read
+    - BYPASS: Bypass cache for this operation
+    """
+
+    ENABLED = "enabled"
+    DISABLED = "disabled"
+    READ_ONLY = "read_only"
+    WRITE_ONLY = "write_only"
+    BYPASS = "bypass"
+
+
+class CacheContext:
+    """
+    Encapsulates cache-related decisions and URL handling.
+
+    This class centralizes all cache-related logic and URL type checking,
+    making the caching behavior more predictable and maintainable.
+
+    Attributes:
+        url (str): The URL being processed.
+        cache_mode (CacheMode): The cache mode for the current operation.
+        always_bypass (bool): If True, bypasses caching for this operation.
+        is_cacheable (bool): True if the URL is cacheable, False otherwise.
+        is_web_url (bool): True if the URL is a web URL, False otherwise.
+        is_local_file (bool): True if the URL is a local file, False otherwise.
+        is_raw_html (bool): True if the URL is raw HTML, False otherwise.
+        _url_display (str): The display name for the URL (web, local file, or raw HTML).
+    """
+
+    def __init__(self, url: str, cache_mode: CacheMode, always_bypass: bool = False):
+        """
+        Initializes the CacheContext with the provided URL and cache mode.
+
+        Args:
+            url (str): The URL being processed.
+            cache_mode (CacheMode): The cache mode for the current operation.
+            always_bypass (bool): If True, bypasses caching for this operation.
+        """
+        self.url = url
+        self.cache_mode = cache_mode
+        self.always_bypass = always_bypass
+        self.is_cacheable = url.startswith(("http://", "https://", "file://"))
+        self.is_web_url = url.startswith(("http://", "https://"))
+        self.is_local_file = url.startswith("file://")
+        self.is_raw_html = url.startswith("raw:")
+        self._url_display = url if not self.is_raw_html else "Raw HTML"
+
+    def should_read(self) -> bool:
+        """
+        Determines if cache should be read based on context.
+
+        How it works:
+        1. If always_bypass is True or is_cacheable is False, return False.
+        2. If cache_mode is ENABLED or READ_ONLY, return True.
+
+        Returns:
+            bool: True if cache should be read, False otherwise.
+        """
+        if self.always_bypass or not self.is_cacheable:
+            return False
+        return self.cache_mode in [CacheMode.ENABLED, CacheMode.READ_ONLY]
+
+    def should_write(self) -> bool:
+        """
+        Determines if cache should be written based on context.
+
+        How it works:
+        1. If always_bypass is True or is_cacheable is False, return False.
+        2. If cache_mode is ENABLED or WRITE_ONLY, return True.
+
+        Returns:
+            bool: True if cache should be written, False otherwise.
+        """
+        if self.always_bypass or not self.is_cacheable:
+            return False
+        return self.cache_mode in [CacheMode.ENABLED, CacheMode.WRITE_ONLY]
+
+    @property
+    def display_url(self) -> str:
+        """Returns the URL in display format."""
+        return self._url_display
+
+
+def _legacy_to_cache_mode(
+    disable_cache: bool = False,
+    bypass_cache: bool = False,
+    no_cache_read: bool = False,
+    no_cache_write: bool = False,
+) -> CacheMode:
+    """
+    Converts legacy cache parameters to the new CacheMode enum.
+
+    This is an internal function to help transition from the old boolean flags
+    to the new CacheMode system.
+    """
+    if disable_cache:
+        return CacheMode.DISABLED
+    if bypass_cache:
+        return CacheMode.BYPASS
+    if no_cache_read and no_cache_write:
+        return CacheMode.DISABLED
+    if no_cache_read:
+        return CacheMode.WRITE_ONLY
+    if no_cache_write:
+        return CacheMode.READ_ONLY
+    return CacheMode.ENABLED

crawl4ai/chunking_strategy.py

@@ -6,19 +6,49 @@ from .model_loader import load_nltk_punkt
 
 # Define the abstract base class for chunking strategies
 class ChunkingStrategy(ABC):
-    
+    """
+    Abstract base class for chunking strategies.
+    """
+
     @abstractmethod
     def chunk(self, text: str) -> list:
         """
         Abstract method to chunk the given text.
+
+        Args:
+            text (str): The text to chunk.
+
+        Returns:
+            list: A list of chunks.
         """
         pass
-    
+
+
+# Create an identity chunking strategy f(x) = [x]
+class IdentityChunking(ChunkingStrategy):
+    """
+    Chunking strategy that returns the input text as a single chunk.
+    """
+
+    def chunk(self, text: str) -> list:
+        return [text]
+
+
 # Regex-based chunking
 class RegexChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text based on regular expression patterns.
+    """
+
     def __init__(self, patterns=None, **kwargs):
+        """
+        Initialize the RegexChunking object.
+
+        Args:
+            patterns (list): A list of regular expression patterns to split text.
+        """
         if patterns is None:
-            patterns = [r'\n\n']  # Default split pattern
+            patterns = [r"\n\n"]  # Default split pattern
         self.patterns = patterns
 
     def chunk(self, text: str) -> list:
@@ -29,12 +59,20 @@ class RegexChunking(ChunkingStrategy):
                 new_paragraphs.extend(re.split(pattern, paragraph))
             paragraphs = new_paragraphs
         return paragraphs
-    
-# NLP-based sentence chunking 
+
+
+# NLP-based sentence chunking
 class NlpSentenceChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into sentences using NLTK's sentence tokenizer.
+    """
+
     def __init__(self, **kwargs):
+        """
+        Initialize the NlpSentenceChunking object.
+        """
+        from crawl4ai.le.legacy.model_loader import load_nltk_punkt
         load_nltk_punkt()
-        pass
 
     def chunk(self, text: str) -> list:
         # Improved regex for sentence splitting
@@ -42,19 +80,35 @@ class NlpSentenceChunking(ChunkingStrategy):
         #     r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<![A-Z][A-Z]\.)(?<![A-Za-z]\.)(?<=\.|\?|\!|\n)\s'
         # )
         # sentences = sentence_endings.split(text)
-        # sens =  [sent.strip() for sent in sentences if sent]            
+        # sens =  [sent.strip() for sent in sentences if sent]
         from nltk.tokenize import sent_tokenize
+
         sentences = sent_tokenize(text)
-        sens =  [sent.strip() for sent in sentences]        
-        
+        sens = [sent.strip() for sent in sentences]
+
         return list(set(sens))
-    
+
+
 # Topic-based segmentation using TextTiling
 class TopicSegmentationChunking(ChunkingStrategy):
-    
+    """
+    Chunking strategy that segments text into topics using NLTK's TextTilingTokenizer.
+
+    How it works:
+    1. Segment the text into topics using TextTilingTokenizer
+    2. Extract keywords for each topic segment
+    """
+
     def __init__(self, num_keywords=3, **kwargs):
+        """
+        Initialize the TopicSegmentationChunking object.
+
+        Args:
+            num_keywords (int): The number of keywords to extract for each topic segment.
+        """
         import nltk as nl
-        self.tokenizer = nl.toknize.TextTilingTokenizer()
+
+        self.tokenizer = nl.tokenize.TextTilingTokenizer()
         self.num_keywords = num_keywords
 
     def chunk(self, text: str) -> list:
@@ -65,8 +119,14 @@ class TopicSegmentationChunking(ChunkingStrategy):
     def extract_keywords(self, text: str) -> list:
         # Tokenize and remove stopwords and punctuation
         import nltk as nl
+
         tokens = nl.toknize.word_tokenize(text)
-        tokens = [token.lower() for token in tokens if token not in nl.corpus.stopwords.words('english') and token not in string.punctuation]
+        tokens = [
+            token.lower()
+            for token in tokens
+            if token not in nl.corpus.stopwords.words("english")
+            and token not in string.punctuation
+        ]
 
         # Calculate frequency distribution
         freq_dist = Counter(tokens)
@@ -77,29 +137,120 @@ class TopicSegmentationChunking(ChunkingStrategy):
         # Segment the text into topics
         segments = self.chunk(text)
         # Extract keywords for each topic segment
-        segments_with_topics = [(segment, self.extract_keywords(segment)) for segment in segments]
+        segments_with_topics = [
+            (segment, self.extract_keywords(segment)) for segment in segments
+        ]
         return segments_with_topics
-    
+
+
 # Fixed-length word chunks
 class FixedLengthWordChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into fixed-length word chunks.
+
+    How it works:
+    1. Split the text into words
+    2. Create chunks of fixed length
+    3. Return the list of chunks
+    """
+
     def __init__(self, chunk_size=100, **kwargs):
+        """
+        Initialize the fixed-length word chunking strategy with the given chunk size.
+
+        Args:
+            chunk_size (int): The size of each chunk in words.
+        """
         self.chunk_size = chunk_size
 
     def chunk(self, text: str) -> list:
         words = text.split()
-        return [' '.join(words[i:i + self.chunk_size]) for i in range(0, len(words), self.chunk_size)]
-    
+        return [
+            " ".join(words[i : i + self.chunk_size])
+            for i in range(0, len(words), self.chunk_size)
+        ]
+
+
 # Sliding window chunking
 class SlidingWindowChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into overlapping word chunks.
+
+    How it works:
+    1. Split the text into words
+    2. Create chunks of fixed length
+    3. Return the list of chunks
+    """
+
     def __init__(self, window_size=100, step=50, **kwargs):
+        """
+        Initialize the sliding window chunking strategy with the given window size and
+        step size.
+
+        Args:
+            window_size (int): The size of the sliding window in words.
+            step (int): The step size for sliding the window in words.
+        """
         self.window_size = window_size
         self.step = step
 
     def chunk(self, text: str) -> list:
         words = text.split()
         chunks = []
-        for i in range(0, len(words), self.step):
-            chunks.append(' '.join(words[i:i + self.window_size]))
-        return chunks
-    
 
+        if len(words) <= self.window_size:
+            return [text]
+
+        for i in range(0, len(words) - self.window_size + 1, self.step):
+            chunk = " ".join(words[i : i + self.window_size])
+            chunks.append(chunk)
+
+        # Handle the last chunk if it doesn't align perfectly
+        if i + self.window_size < len(words):
+            chunks.append(" ".join(words[-self.window_size :]))
+
+        return chunks
+
+
+class OverlappingWindowChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into overlapping word chunks.
+
+    How it works:
+    1. Split the text into words using whitespace
+    2. Create chunks of fixed length equal to the window size
+    3. Slide the window by the overlap size
+    4. Return the list of chunks
+    """
+
+    def __init__(self, window_size=1000, overlap=100, **kwargs):
+        """
+        Initialize the overlapping window chunking strategy with the given window size and
+        overlap size.
+
+        Args:
+            window_size (int): The size of the window in words.
+            overlap (int): The size of the overlap between consecutive chunks in words.
+        """
+        self.window_size = window_size
+        self.overlap = overlap
+
+    def chunk(self, text: str) -> list:
+        words = text.split()
+        chunks = []
+
+        if len(words) <= self.window_size:
+            return [text]
+
+        start = 0
+        while start < len(words):
+            end = start + self.window_size
+            chunk = " ".join(words[start:end])
+            chunks.append(chunk)
+
+            if end >= len(words):
+                break
+
+            start = end - self.overlap
+
+        return chunks

crawl4ai/components/crawler_monitor.py

@@ -0,0 +1,837 @@
+import time
+import uuid
+import threading
+import psutil
+from datetime import datetime, timedelta
+from typing import Dict, Optional, List
+import threading
+from rich.console import Console
+from rich.layout import Layout
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+from rich.live import Live
+from rich import box
+from ..models import CrawlStatus
+
+class TerminalUI:
+    """Terminal user interface for CrawlerMonitor using rich library."""
+    
+    def __init__(self, refresh_rate: float = 1.0, max_width: int = 120):
+        """
+        Initialize the terminal UI.
+        
+        Args:
+            refresh_rate: How often to refresh the UI (in seconds)
+            max_width: Maximum width of the UI in characters
+        """
+        self.console = Console(width=max_width)
+        self.layout = Layout()
+        self.refresh_rate = refresh_rate
+        self.stop_event = threading.Event()
+        self.ui_thread = None
+        self.monitor = None  # Will be set by CrawlerMonitor
+        self.max_width = max_width
+        
+        # Setup layout - vertical layout (top to bottom)
+        self.layout.split(
+            Layout(name="header", size=3),
+            Layout(name="pipeline_status", size=10),
+            Layout(name="task_details", ratio=1),
+            Layout(name="footer", size=3)  # Increased footer size to fit all content
+        )
+        
+    def start(self, monitor):
+        """Start the UI thread."""
+        self.monitor = monitor
+        self.stop_event.clear()
+        self.ui_thread = threading.Thread(target=self._ui_loop)
+        self.ui_thread.daemon = True
+        self.ui_thread.start()
+        
+    def stop(self):
+        """Stop the UI thread."""
+        if self.ui_thread and self.ui_thread.is_alive():
+            self.stop_event.set()
+            # Only try to join if we're not in the UI thread
+            # This prevents "cannot join current thread" errors
+            if threading.current_thread() != self.ui_thread:
+                self.ui_thread.join(timeout=5.0)
+    
+    def _ui_loop(self):
+        """Main UI rendering loop."""
+        import sys
+        import select
+        import termios
+        import tty
+        
+        # Setup terminal for non-blocking input
+        old_settings = termios.tcgetattr(sys.stdin)
+        try:
+            tty.setcbreak(sys.stdin.fileno())
+            
+            # Use Live display to render the UI
+            with Live(self.layout, refresh_per_second=1/self.refresh_rate, screen=True) as live:
+                self.live = live  # Store the live display for updates
+                
+                # Main UI loop
+                while not self.stop_event.is_set():
+                    self._update_display()
+                    
+                    # Check for key press (non-blocking)
+                    if select.select([sys.stdin], [], [], 0)[0]:
+                        key = sys.stdin.read(1)
+                        # Check for 'q' to quit
+                        if key == 'q':
+                            # Signal stop but don't call monitor.stop() from UI thread
+                            # as it would cause the thread to try to join itself
+                            self.stop_event.set()
+                            self.monitor.is_running = False
+                            break
+                    
+                    time.sleep(self.refresh_rate)
+                    
+                    # Just check if the monitor was stopped
+                    if not self.monitor.is_running:
+                        break
+        finally:
+            # Restore terminal settings
+            termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
+    
+    def _update_display(self):
+        """Update the terminal display with current statistics."""
+        if not self.monitor:
+            return
+            
+        # Update crawler status panel
+        self.layout["header"].update(self._create_status_panel())
+        
+        # Update pipeline status panel and task details panel
+        self.layout["pipeline_status"].update(self._create_pipeline_panel())
+        self.layout["task_details"].update(self._create_task_details_panel())
+        
+        # Update footer
+        self.layout["footer"].update(self._create_footer())
+    
+    def _create_status_panel(self) -> Panel:
+        """Create the crawler status panel."""
+        summary = self.monitor.get_summary()
+        
+        # Format memory status with icon
+        memory_status = self.monitor.get_memory_status()
+        memory_icon = "🟢"  # Default NORMAL
+        if memory_status == "PRESSURE":
+            memory_icon = "🟠"
+        elif memory_status == "CRITICAL":
+            memory_icon = "🔴"
+        
+        # Get current memory usage
+        current_memory = psutil.Process().memory_info().rss / (1024 * 1024)  # MB
+        memory_percent = (current_memory / psutil.virtual_memory().total) * 100
+        
+        # Format runtime
+        runtime = self.monitor._format_time(time.time() - self.monitor.start_time if self.monitor.start_time else 0)
+        
+        # Create the status text
+        status_text = Text()
+        status_text.append(f"Web Crawler Dashboard | Runtime: {runtime} | Memory: {memory_percent:.1f}% {memory_icon}\n")
+        status_text.append(f"Status: {memory_status} | URLs: {summary['urls_completed']}/{summary['urls_total']} | ")
+        status_text.append(f"Peak Mem: {summary['peak_memory_percent']:.1f}% at {self.monitor._format_time(summary['peak_memory_time'])}")
+        
+        return Panel(status_text, title="Crawler Status", border_style="blue")
+    
+    def _create_pipeline_panel(self) -> Panel:
+        """Create the pipeline status panel."""
+        summary = self.monitor.get_summary()
+        queue_stats = self.monitor.get_queue_stats()
+        
+        # Create a table for status counts
+        table = Table(show_header=True, box=None)
+        table.add_column("Status", style="cyan")
+        table.add_column("Count", justify="right")
+        table.add_column("Percentage", justify="right")
+        table.add_column("Stat", style="cyan")
+        table.add_column("Value", justify="right")
+        
+        # Calculate overall progress
+        progress = f"{summary['urls_completed']}/{summary['urls_total']}"
+        progress_percent = f"{summary['completion_percentage']:.1f}%"
+        
+        # Add rows for each status
+        table.add_row(
+            "Overall Progress", 
+            progress, 
+            progress_percent,
+            "Est. Completion", 
+            summary.get('estimated_completion_time', "N/A")
+        )
+        
+        # Add rows for each status
+        status_counts = summary['status_counts']
+        total = summary['urls_total'] or 1  # Avoid division by zero
+        
+        # Status rows
+        table.add_row(
+            "Completed", 
+            str(status_counts.get(CrawlStatus.COMPLETED.name, 0)),
+            f"{status_counts.get(CrawlStatus.COMPLETED.name, 0) / total * 100:.1f}%",
+            "Avg. Time/URL",
+            f"{summary.get('avg_task_duration', 0):.2f}s"
+        )
+        
+        table.add_row(
+            "Failed", 
+            str(status_counts.get(CrawlStatus.FAILED.name, 0)),
+            f"{status_counts.get(CrawlStatus.FAILED.name, 0) / total * 100:.1f}%",
+            "Concurrent Tasks",
+            str(status_counts.get(CrawlStatus.IN_PROGRESS.name, 0))
+        )
+        
+        table.add_row(
+            "In Progress", 
+            str(status_counts.get(CrawlStatus.IN_PROGRESS.name, 0)),
+            f"{status_counts.get(CrawlStatus.IN_PROGRESS.name, 0) / total * 100:.1f}%",
+            "Queue Size",
+            str(queue_stats['total_queued'])
+        )
+        
+        table.add_row(
+            "Queued", 
+            str(status_counts.get(CrawlStatus.QUEUED.name, 0)),
+            f"{status_counts.get(CrawlStatus.QUEUED.name, 0) / total * 100:.1f}%",
+            "Max Wait Time",
+            f"{queue_stats['highest_wait_time']:.1f}s"
+        )
+        
+        # Requeued is a special case as it's not a status
+        requeued_count = summary.get('requeued_count', 0)
+        table.add_row(
+            "Requeued", 
+            str(requeued_count),
+            f"{summary.get('requeue_rate', 0):.1f}%",
+            "Avg Wait Time",
+            f"{queue_stats['avg_wait_time']:.1f}s"
+        )
+        
+        # Add empty row for spacing
+        table.add_row(
+            "", 
+            "",
+            "",
+            "Requeue Rate",
+            f"{summary.get('requeue_rate', 0):.1f}%"
+        )
+        
+        return Panel(table, title="Pipeline Status", border_style="green")
+    
+    def _create_task_details_panel(self) -> Panel:
+        """Create the task details panel."""
+        # Create a table for task details
+        table = Table(show_header=True, expand=True)
+        table.add_column("Task ID", style="cyan", no_wrap=True, width=10)
+        table.add_column("URL", style="blue", ratio=3)
+        table.add_column("Status", style="green", width=15)
+        table.add_column("Memory", justify="right", width=8)
+        table.add_column("Peak", justify="right", width=8)
+        table.add_column("Duration", justify="right", width=10)
+        
+        # Get all task stats
+        task_stats = self.monitor.get_all_task_stats()
+        
+        # Add summary row
+        active_tasks = sum(1 for stats in task_stats.values() 
+                          if stats['status'] == CrawlStatus.IN_PROGRESS.name)
+        
+        total_memory = sum(stats['memory_usage'] for stats in task_stats.values())
+        total_peak = sum(stats['peak_memory'] for stats in task_stats.values())
+        
+        # Summary row with separators
+        table.add_row(
+            "SUMMARY", 
+            f"Total: {len(task_stats)}", 
+            f"Active: {active_tasks}",
+            f"{total_memory:.1f}",
+            f"{total_peak:.1f}",
+            "N/A"
+        )
+        
+        # Add a separator
+        table.add_row("—" * 10, "—" * 20, "—" * 10, "—" * 8, "—" * 8, "—" * 10)
+        
+        # Status icons
+        status_icons = {
+            CrawlStatus.QUEUED.name: "⏳",
+            CrawlStatus.IN_PROGRESS.name: "🔄",
+            CrawlStatus.COMPLETED.name: "✅",
+            CrawlStatus.FAILED.name: "❌"
+        }
+        
+        # Calculate how many rows we can display based on available space
+        # We can display more rows now that we have a dedicated panel
+        display_count = min(len(task_stats), 20)  # Display up to 20 tasks
+        
+        # Add rows for each task
+        for task_id, stats in sorted(
+            list(task_stats.items())[:display_count],
+            # Sort: 1. IN_PROGRESS first, 2. QUEUED, 3. COMPLETED/FAILED by recency
+            key=lambda x: (
+                0 if x[1]['status'] == CrawlStatus.IN_PROGRESS.name else 
+                1 if x[1]['status'] == CrawlStatus.QUEUED.name else 
+                2,
+                -1 * (x[1].get('end_time', 0) or 0)  # Most recent first
+            )
+        ):
+            # Truncate task_id and URL for display
+            short_id = task_id[:8]
+            url = stats['url']
+            if len(url) > 50:  # Allow longer URLs in the dedicated panel
+                url = url[:47] + "..."
+                
+            # Format status with icon
+            status = f"{status_icons.get(stats['status'], '?')} {stats['status']}"
+            
+            # Add row
+            table.add_row(
+                short_id,
+                url,
+                status,
+                f"{stats['memory_usage']:.1f}",
+                f"{stats['peak_memory']:.1f}",
+                stats['duration'] if 'duration' in stats else "0:00"
+            )
+        
+        return Panel(table, title="Task Details", border_style="yellow")
+    
+    def _create_footer(self) -> Panel:
+        """Create the footer panel."""
+        from rich.columns import Columns
+        from rich.align import Align
+        
+        memory_status = self.monitor.get_memory_status()
+        memory_icon = "🟢"  # Default NORMAL
+        if memory_status == "PRESSURE":
+            memory_icon = "🟠"
+        elif memory_status == "CRITICAL":
+            memory_icon = "🔴"
+        
+        # Left section - memory status
+        left_text = Text()
+        left_text.append("Memory Status: ", style="bold")
+        status_style = "green" if memory_status == "NORMAL" else "yellow" if memory_status == "PRESSURE" else "red bold"
+        left_text.append(f"{memory_icon} {memory_status}", style=status_style)
+        
+        # Center section - copyright
+        center_text = Text("© Crawl4AI 2025 | Made by UnclecCode", style="cyan italic")
+        
+        # Right section - quit instruction
+        right_text = Text()
+        right_text.append("Press ", style="bold")
+        right_text.append("q", style="white on blue")
+        right_text.append(" to quit", style="bold")
+        
+        # Create columns with the three sections
+        footer_content = Columns(
+            [
+                Align.left(left_text),
+                Align.center(center_text),
+                Align.right(right_text)
+            ],
+            expand=True
+        )
+        
+        # Create a more visible footer panel
+        return Panel(
+            footer_content,
+            border_style="white",
+            padding=(0, 1)  # Add padding for better visibility
+        )
+
+
+class CrawlerMonitor:
+    """
+    Comprehensive monitoring and visualization system for tracking web crawler operations in real-time.
+    Provides a terminal-based dashboard that displays task statuses, memory usage, queue statistics,
+    and performance metrics.
+    """
+    
+    def __init__(
+        self,
+        urls_total: int = 0,
+        refresh_rate: float = 1.0,
+        enable_ui: bool = True,
+        max_width: int = 120
+    ):
+        """
+        Initialize the CrawlerMonitor.
+        
+        Args:
+            urls_total: Total number of URLs to be crawled
+            refresh_rate: How often to refresh the UI (in seconds)
+            enable_ui: Whether to display the terminal UI
+            max_width: Maximum width of the UI in characters
+        """
+        # Core monitoring attributes
+        self.stats = {}  # Task ID -> stats dict
+        self.memory_status = "NORMAL"
+        self.start_time = None
+        self.end_time = None
+        self.is_running = False
+        self.queue_stats = {
+            "total_queued": 0,
+            "highest_wait_time": 0.0,
+            "avg_wait_time": 0.0
+        }
+        self.urls_total = urls_total
+        self.urls_completed = 0
+        self.peak_memory_percent = 0.0
+        self.peak_memory_time = 0.0
+        
+        # Status counts
+        self.status_counts = {
+            CrawlStatus.QUEUED.name: 0,
+            CrawlStatus.IN_PROGRESS.name: 0,
+            CrawlStatus.COMPLETED.name: 0,
+            CrawlStatus.FAILED.name: 0
+        }
+        
+        # Requeue tracking
+        self.requeued_count = 0
+        
+        # Thread-safety
+        self._lock = threading.RLock()
+        
+        # Terminal UI
+        self.enable_ui = enable_ui
+        self.terminal_ui = TerminalUI(
+            refresh_rate=refresh_rate, 
+            max_width=max_width
+        ) if enable_ui else None
+    
+    def start(self):
+        """
+        Start the monitoring session.
+        
+        - Initializes the start_time
+        - Sets is_running to True
+        - Starts the terminal UI if enabled
+        """
+        with self._lock:
+            self.start_time = time.time()
+            self.is_running = True
+            
+            # Start the terminal UI
+            if self.enable_ui and self.terminal_ui:
+                self.terminal_ui.start(self)
+    
+    def stop(self):
+        """
+        Stop the monitoring session.
+        
+        - Records end_time
+        - Sets is_running to False
+        - Stops the terminal UI
+        - Generates final summary statistics
+        """
+        with self._lock:
+            self.end_time = time.time()
+            self.is_running = False
+            
+            # Stop the terminal UI
+            if self.enable_ui and self.terminal_ui:
+                self.terminal_ui.stop()
+    
+    def add_task(self, task_id: str, url: str):
+        """
+        Register a new task with the monitor.
+        
+        Args:
+            task_id: Unique identifier for the task
+            url: URL being crawled
+            
+        The task is initialized with:
+            - status: QUEUED
+            - url: The URL to crawl
+            - enqueue_time: Current time
+            - memory_usage: 0
+            - peak_memory: 0
+            - wait_time: 0
+            - retry_count: 0
+        """
+        with self._lock:
+            self.stats[task_id] = {
+                "task_id": task_id,
+                "url": url,
+                "status": CrawlStatus.QUEUED.name,
+                "enqueue_time": time.time(),
+                "start_time": None,
+                "end_time": None,
+                "memory_usage": 0.0,
+                "peak_memory": 0.0,
+                "error_message": "",
+                "wait_time": 0.0,
+                "retry_count": 0,
+                "duration": "0:00",
+                "counted_requeue": False
+            }
+            
+            # Update status counts
+            self.status_counts[CrawlStatus.QUEUED.name] += 1
+    
+    def update_task(
+        self, 
+        task_id: str, 
+        status: Optional[CrawlStatus] = None,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None,
+        memory_usage: Optional[float] = None,
+        peak_memory: Optional[float] = None,
+        error_message: Optional[str] = None,
+        retry_count: Optional[int] = None,
+        wait_time: Optional[float] = None
+    ):
+        """
+        Update statistics for a specific task.
+        
+        Args:
+            task_id: Unique identifier for the task
+            status: New status (QUEUED, IN_PROGRESS, COMPLETED, FAILED)
+            start_time: When task execution started
+            end_time: When task execution ended
+            memory_usage: Current memory usage in MB
+            peak_memory: Maximum memory usage in MB
+            error_message: Error description if failed
+            retry_count: Number of retry attempts
+            wait_time: Time spent in queue
+            
+        Updates task statistics and updates status counts.
+        If status changes, decrements old status count and 
+        increments new status count.
+        """
+        with self._lock:
+            # Check if task exists
+            if task_id not in self.stats:
+                return
+            
+            task_stats = self.stats[task_id]
+            
+            # Update status counts if status is changing
+            old_status = task_stats["status"]
+            if status and status.name != old_status:
+                self.status_counts[old_status] -= 1
+                self.status_counts[status.name] += 1
+                
+                # Track completion
+                if status == CrawlStatus.COMPLETED:
+                    self.urls_completed += 1
+                
+                # Track requeues
+                if old_status in [CrawlStatus.COMPLETED.name, CrawlStatus.FAILED.name] and not task_stats.get("counted_requeue", False):
+                    self.requeued_count += 1
+                    task_stats["counted_requeue"] = True
+            
+            # Update task statistics
+            if status:
+                task_stats["status"] = status.name
+            if start_time is not None:
+                task_stats["start_time"] = start_time
+            if end_time is not None:
+                task_stats["end_time"] = end_time
+            if memory_usage is not None:
+                task_stats["memory_usage"] = memory_usage
+                
+                # Update peak memory if necessary
+                current_percent = (memory_usage / psutil.virtual_memory().total) * 100
+                if current_percent > self.peak_memory_percent:
+                    self.peak_memory_percent = current_percent
+                    self.peak_memory_time = time.time()
+                
+            if peak_memory is not None:
+                task_stats["peak_memory"] = peak_memory
+            if error_message is not None:
+                task_stats["error_message"] = error_message
+            if retry_count is not None:
+                task_stats["retry_count"] = retry_count
+            if wait_time is not None:
+                task_stats["wait_time"] = wait_time
+            
+            # Calculate duration
+            if task_stats["start_time"]:
+                end = task_stats["end_time"] or time.time()
+                duration = end - task_stats["start_time"]
+                task_stats["duration"] = self._format_time(duration)
+    
+    def update_memory_status(self, status: str):
+        """
+        Update the current memory status.
+        
+        Args:
+            status: Memory status (NORMAL, PRESSURE, CRITICAL, or custom)
+            
+        Also updates the UI to reflect the new status.
+        """
+        with self._lock:
+            self.memory_status = status
+    
+    def update_queue_statistics(
+        self,
+        total_queued: int,
+        highest_wait_time: float,
+        avg_wait_time: float
+    ):
+        """
+        Update statistics related to the task queue.
+        
+        Args:
+            total_queued: Number of tasks currently in queue
+            highest_wait_time: Longest wait time of any queued task
+            avg_wait_time: Average wait time across all queued tasks
+        """
+        with self._lock:
+            self.queue_stats = {
+                "total_queued": total_queued,
+                "highest_wait_time": highest_wait_time,
+                "avg_wait_time": avg_wait_time
+            }
+    
+    def get_task_stats(self, task_id: str) -> Dict:
+        """
+        Get statistics for a specific task.
+        
+        Args:
+            task_id: Unique identifier for the task
+            
+        Returns:
+            Dictionary containing all task statistics
+        """
+        with self._lock:
+            return self.stats.get(task_id, {}).copy()
+    
+    def get_all_task_stats(self) -> Dict[str, Dict]:
+        """
+        Get statistics for all tasks.
+        
+        Returns:
+            Dictionary mapping task_ids to their statistics
+        """
+        with self._lock:
+            return self.stats.copy()
+    
+    def get_memory_status(self) -> str:
+        """
+        Get the current memory status.
+        
+        Returns:
+            Current memory status string
+        """
+        with self._lock:
+            return self.memory_status
+    
+    def get_queue_stats(self) -> Dict:
+        """
+        Get current queue statistics.
+        
+        Returns:
+            Dictionary with queue statistics including:
+            - total_queued: Number of tasks in queue
+            - highest_wait_time: Longest wait time
+            - avg_wait_time: Average wait time
+        """
+        with self._lock:
+            return self.queue_stats.copy()
+    
+    def get_summary(self) -> Dict:
+        """
+        Get a summary of all crawler statistics.
+        
+        Returns:
+            Dictionary containing:
+            - runtime: Total runtime in seconds
+            - urls_total: Total URLs to process
+            - urls_completed: Number of completed URLs
+            - completion_percentage: Percentage complete
+            - status_counts: Count of tasks in each status
+            - memory_status: Current memory status
+            - peak_memory_percent: Highest memory usage
+            - peak_memory_time: When peak memory occurred
+            - avg_task_duration: Average task processing time
+            - estimated_completion_time: Projected finish time
+            - requeue_rate: Percentage of tasks requeued
+        """
+        with self._lock:
+            # Calculate runtime
+            current_time = time.time()
+            runtime = current_time - (self.start_time or current_time)
+            
+            # Calculate completion percentage
+            completion_percentage = 0
+            if self.urls_total > 0:
+                completion_percentage = (self.urls_completed / self.urls_total) * 100
+            
+            # Calculate average task duration for completed tasks
+            completed_tasks = [
+                task for task in self.stats.values() 
+                if task["status"] == CrawlStatus.COMPLETED.name and task.get("start_time") and task.get("end_time")
+            ]
+            
+            avg_task_duration = 0
+            if completed_tasks:
+                total_duration = sum(task["end_time"] - task["start_time"] for task in completed_tasks)
+                avg_task_duration = total_duration / len(completed_tasks)
+            
+            # Calculate requeue rate
+            requeue_rate = 0
+            if len(self.stats) > 0:
+                requeue_rate = (self.requeued_count / len(self.stats)) * 100
+            
+            # Calculate estimated completion time
+            estimated_completion_time = "N/A"
+            if avg_task_duration > 0 and self.urls_total > 0 and self.urls_completed > 0:
+                remaining_tasks = self.urls_total - self.urls_completed
+                estimated_seconds = remaining_tasks * avg_task_duration
+                estimated_completion_time = self._format_time(estimated_seconds)
+            
+            return {
+                "runtime": runtime,
+                "urls_total": self.urls_total,
+                "urls_completed": self.urls_completed,
+                "completion_percentage": completion_percentage,
+                "status_counts": self.status_counts.copy(),
+                "memory_status": self.memory_status,
+                "peak_memory_percent": self.peak_memory_percent,
+                "peak_memory_time": self.peak_memory_time,
+                "avg_task_duration": avg_task_duration,
+                "estimated_completion_time": estimated_completion_time,
+                "requeue_rate": requeue_rate,
+                "requeued_count": self.requeued_count
+            }
+    
+    def render(self):
+        """
+        Render the terminal UI.
+        
+        This is the main UI rendering loop that:
+        1. Updates all statistics
+        2. Formats the display
+        3. Renders the ASCII interface
+        4. Handles keyboard input
+        
+        Note: The actual rendering is handled by the TerminalUI class
+        which uses the rich library's Live display.
+        """
+        if self.enable_ui and self.terminal_ui:
+            # Force an update of the UI
+            if hasattr(self.terminal_ui, '_update_display'):
+                self.terminal_ui._update_display()
+    
+    def _format_time(self, seconds: float) -> str:
+        """
+        Format time in hours:minutes:seconds.
+        
+        Args:
+            seconds: Time in seconds
+            
+        Returns:
+            Formatted time string (e.g., "1:23:45")
+        """
+        delta = timedelta(seconds=int(seconds))
+        hours, remainder = divmod(delta.seconds, 3600)
+        minutes, seconds = divmod(remainder, 60)
+        
+        if hours > 0:
+            return f"{hours}:{minutes:02}:{seconds:02}"
+        else:
+            return f"{minutes}:{seconds:02}"
+    
+    def _calculate_estimated_completion(self) -> str:
+        """
+        Calculate estimated completion time based on current progress.
+        
+        Returns:
+            Formatted time string
+        """
+        summary = self.get_summary()
+        return summary.get("estimated_completion_time", "N/A")
+
+
+# Example code for testing
+if __name__ == "__main__":
+    # Initialize the monitor
+    monitor = CrawlerMonitor(urls_total=100)
+    
+    # Start monitoring
+    monitor.start()
+    
+    try:
+        # Simulate some tasks
+        for i in range(20):
+            task_id = str(uuid.uuid4())
+            url = f"https://example.com/page{i}"
+            monitor.add_task(task_id, url)
+            
+            # Simulate 20% of tasks are already running
+            if i < 4:
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.IN_PROGRESS,
+                    start_time=time.time() - 30,  # Started 30 seconds ago
+                    memory_usage=10.5
+                )
+                
+            # Simulate 10% of tasks are completed
+            if i >= 4 and i < 6:
+                start_time = time.time() - 60
+                end_time = time.time() - 15
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.IN_PROGRESS,
+                    start_time=start_time,
+                    memory_usage=8.2
+                )
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.COMPLETED,
+                    end_time=end_time,
+                    memory_usage=0,
+                    peak_memory=15.7
+                )
+                
+            # Simulate 5% of tasks fail
+            if i >= 6 and i < 7:
+                start_time = time.time() - 45
+                end_time = time.time() - 20
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.IN_PROGRESS,
+                    start_time=start_time,
+                    memory_usage=12.3
+                )
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.FAILED,
+                    end_time=end_time,
+                    memory_usage=0,
+                    peak_memory=18.2,
+                    error_message="Connection timeout"
+                )
+        
+        # Simulate memory pressure
+        monitor.update_memory_status("PRESSURE")
+        
+        # Simulate queue statistics
+        monitor.update_queue_statistics(
+            total_queued=16,  # 20 - 4 (in progress)
+            highest_wait_time=120.5,
+            avg_wait_time=60.2
+        )
+        
+        # Keep the monitor running for a demonstration
+        print("Crawler Monitor is running. Press 'q' to exit.")
+        while monitor.is_running:
+            time.sleep(0.1)
+            
+    except KeyboardInterrupt:
+        print("\nExiting crawler monitor...")
+    finally:
+        # Stop the monitor
+        monitor.stop()
+        print("Crawler monitor exited successfully.")

crawl4ai/config.py

@@ -4,26 +4,135 @@ from dotenv import load_dotenv
 load_dotenv()  # Load environment variables from .env file
 
 # Default provider, ONLY used when the extraction strategy is LLMExtractionStrategy
-DEFAULT_PROVIDER = "openai/gpt-4-turbo"
+DEFAULT_PROVIDER = "openai/gpt-4o"
+DEFAULT_PROVIDER_API_KEY = "OPENAI_API_KEY"
 MODEL_REPO_BRANCH = "new-release-0.0.2"
 # Provider-model dictionary, ONLY used when the extraction strategy is LLMExtractionStrategy
 PROVIDER_MODELS = {
-    "ollama/llama3": "no-token-needed", # Any model from Ollama no need for API token
+    "ollama/llama3": "no-token-needed",  # Any model from Ollama no need for API token
     "groq/llama3-70b-8192": os.getenv("GROQ_API_KEY"),
     "groq/llama3-8b-8192": os.getenv("GROQ_API_KEY"),
-    "openai/gpt-3.5-turbo": os.getenv("OPENAI_API_KEY"),
-    "openai/gpt-4-turbo": os.getenv("OPENAI_API_KEY"),
+    "openai/gpt-4o-mini": os.getenv("OPENAI_API_KEY"),
     "openai/gpt-4o": os.getenv("OPENAI_API_KEY"),
+    "openai/o1-mini": os.getenv("OPENAI_API_KEY"),
+    "openai/o1-preview": os.getenv("OPENAI_API_KEY"),
+    "openai/o3-mini": os.getenv("OPENAI_API_KEY"),
+    "openai/o3-mini-high": os.getenv("OPENAI_API_KEY"),
     "anthropic/claude-3-haiku-20240307": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-opus-20240229": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-sonnet-20240229": os.getenv("ANTHROPIC_API_KEY"),
+    "anthropic/claude-3-5-sonnet-20240620": os.getenv("ANTHROPIC_API_KEY"),
+    "gemini/gemini-pro": os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-1.5-pro': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash-exp': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash-lite-preview-02-05': os.getenv("GEMINI_API_KEY"),
+    "deepseek/deepseek-chat": os.getenv("DEEPSEEK_API_KEY"),
 }
 
-
 # Chunk token threshold
-CHUNK_TOKEN_THRESHOLD = 500
+CHUNK_TOKEN_THRESHOLD = 2**11  # 2048 tokens
 OVERLAP_RATE = 0.1
 WORD_TOKEN_RATE = 1.3
 
-# Threshold for the minimum number of word in a HTML tag to be considered 
+# Threshold for the minimum number of word in a HTML tag to be considered
 MIN_WORD_THRESHOLD = 1
+IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD = 1
+
+IMPORTANT_ATTRS = ["src", "href", "alt", "title", "width", "height"]
+ONLY_TEXT_ELIGIBLE_TAGS = [
+    "b",
+    "i",
+    "u",
+    "span",
+    "del",
+    "ins",
+    "sub",
+    "sup",
+    "strong",
+    "em",
+    "code",
+    "kbd",
+    "var",
+    "s",
+    "q",
+    "abbr",
+    "cite",
+    "dfn",
+    "time",
+    "small",
+    "mark",
+]
+SOCIAL_MEDIA_DOMAINS = [
+    "facebook.com",
+    "twitter.com",
+    "x.com",
+    "linkedin.com",
+    "instagram.com",
+    "pinterest.com",
+    "tiktok.com",
+    "snapchat.com",
+    "reddit.com",
+]
+
+# Threshold for the Image extraction - Range is 1 to 6
+# Images are scored based on point based system, to filter based on usefulness. Points are assigned
+# to each image based on the following aspects.
+# If either height or width exceeds 150px
+# If image size is greater than 10Kb
+# If alt property is set
+# If image format is in jpg, png or webp
+# If image is in the first half of the total images extracted from the page
+IMAGE_SCORE_THRESHOLD = 2
+
+MAX_METRICS_HISTORY = 1000
+
+NEED_MIGRATION = True
+URL_LOG_SHORTEN_LENGTH = 30
+SHOW_DEPRECATION_WARNINGS = True
+SCREENSHOT_HEIGHT_TRESHOLD = 10000
+PAGE_TIMEOUT = 60000
+DOWNLOAD_PAGE_TIMEOUT = 60000
+
+# Global user settings with descriptions and default values
+USER_SETTINGS = {
+    "DEFAULT_LLM_PROVIDER": {
+        "default": "openai/gpt-4o",
+        "description": "Default LLM provider in 'company/model' format (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')",
+        "type": "string"
+    },
+    "DEFAULT_LLM_PROVIDER_TOKEN": {
+        "default": "",
+        "description": "API token for the default LLM provider",
+        "type": "string",
+        "secret": True
+    },
+    "VERBOSE": {
+        "default": False,
+        "description": "Enable verbose output for all commands",
+        "type": "boolean"
+    },
+    "BROWSER_HEADLESS": {
+        "default": True,
+        "description": "Run browser in headless mode by default",
+        "type": "boolean"
+    },
+    "BROWSER_TYPE": {
+        "default": "chromium",
+        "description": "Default browser type (chromium or firefox)",
+        "type": "string",
+        "options": ["chromium", "firefox"]
+    },
+    "CACHE_MODE": {
+        "default": "bypass",
+        "description": "Default cache mode (bypass, use, or refresh)",
+        "type": "string",
+        "options": ["bypass", "use", "refresh"]
+    },
+    "USER_AGENT_MODE": {
+        "default": "default",
+        "description": "Default user agent mode (default, random, or mobile)",
+        "type": "string",
+        "options": ["default", "random", "mobile"]
+    }
+}

crawl4ai/crawlers/amazon_product/crawler.py

@@ -0,0 +1,20 @@
+from crawl4ai.hub import BaseCrawler
+
+__meta__ = {
+    "version": "1.2.0",
+    "tested_on": ["amazon.com"],
+    "rate_limit": "50 RPM",
+    "schema": {"product": ["name", "price"]}
+}
+
+class AmazonProductCrawler(BaseCrawler):
+    async def run(self, url: str, **kwargs) -> str:
+        try:
+            self.logger.info(f"Crawling {url}")
+            return '{"product": {"name": "Test Amazon Product"}}'
+        except Exception as e:
+            self.logger.error(f"Crawl failed: {str(e)}")
+            return json.dumps({
+                "error": str(e),
+                "metadata": self.meta  # Include meta in error response
+            })            

crawl4ai/crawlers/google_search/crawler.py

@@ -0,0 +1,131 @@
+from crawl4ai import BrowserConfig, AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai.hub import BaseCrawler
+from crawl4ai.utils import optimize_html, get_home_folder, preprocess_html_for_schema
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from pathlib import Path
+import json
+import os
+from typing import Dict
+
+
+class GoogleSearchCrawler(BaseCrawler):
+    __meta__ = {
+        "version": "1.0.0",
+        "tested_on": ["google.com/search*"],
+        "rate_limit": "10 RPM",
+        "description": "Crawls Google Search results (text + images)",
+    }
+
+    def __init__(self):
+        super().__init__()
+        self.js_script = (Path(__file__).parent /
+                          "script.js").read_text()
+
+    async def run(self, url="", query: str = "", search_type: str = "text", schema_cache_path = None, **kwargs) -> str:
+        """Crawl Google Search results for a query"""
+        url = f"https://www.google.com/search?q={query}&gl=sg&hl=en" if search_type == "text" else f"https://www.google.com/search?q={query}&gl=sg&hl=en&tbs=qdr:d&udm=2"
+        if kwargs.get("page_start", 1) > 1:
+            url = f"{url}&start={kwargs['page_start'] * 10}"
+        if kwargs.get("page_length", 1) > 1:
+            url = f"{url}&num={kwargs['page_length']}"
+            
+        browser_config = BrowserConfig(headless=True, verbose=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            config = CrawlerRunConfig(
+                cache_mode=kwargs.get("cache_mode", CacheMode.BYPASS),
+                keep_attrs=["id", "class"],
+                keep_data_attributes=True,
+                delay_before_return_html=kwargs.get(
+                    "delay", 2 if search_type == "image" else 1),
+                js_code=self.js_script if search_type == "image" else None,
+            )
+
+            result = await crawler.arun(url=url, config=config)
+            if not result.success:
+                return json.dumps({"error": result.error})
+
+            if search_type == "image":
+                if result.js_execution_result.get("success", False) is False:
+                    return json.dumps({"error": result.js_execution_result.get("error", "Unknown error")})
+                if "results" in result.js_execution_result:
+                    image_result = result.js_execution_result['results'][0]
+                    if image_result.get("success", False) is False:
+                        return json.dumps({"error": image_result.get("error", "Unknown error")})
+                    return json.dumps(image_result["result"], indent=4)
+
+            # For text search, extract structured data
+            schemas = await self._build_schemas(result.cleaned_html, schema_cache_path)
+            extracted = {
+                key: JsonCssExtractionStrategy(schema=schemas[key]).run(
+                    url=url, sections=[result.html]
+                )
+                for key in schemas
+            }
+            return json.dumps(extracted, indent=4)
+
+    async def _build_schemas(self, html: str, schema_cache_path: str = None) -> Dict[str, Dict]:
+        """Build extraction schemas (organic, top stories, etc.)"""
+        home_dir = get_home_folder() if not schema_cache_path else schema_cache_path
+        os.makedirs(f"{home_dir}/schema", exist_ok=True)
+
+        # cleaned_html = optimize_html(html, threshold=100)
+        cleaned_html = preprocess_html_for_schema(html) 
+
+        organic_schema = None
+        if os.path.exists(f"{home_dir}/schema/organic_schema.json"):
+            with open(f"{home_dir}/schema/organic_schema.json", "r") as f:
+                organic_schema = json.load(f)
+        else:
+            organic_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "snippet": "...",
+            "date": "1 hour ago",
+        }""",
+                query="""The given html is the crawled html from Google search result. Please find the schema for organic search item in the given html, I am interested in title, link, snippet text. date."""
+            )
+
+            with open(f"{home_dir}/schema/organic_schema.json", "w") as f:
+                f.write(json.dumps(organic_schema))
+
+        top_stories_schema = None
+        if os.path.exists(f"{home_dir}/schema/top_stories_schema.json"):
+            with open(f"{home_dir}/schema/top_stories_schema.json", "r") as f:
+                top_stories_schema = json.load(f)
+        else:
+            top_stories_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "source": "Insider Monkey",
+            "date": "1 hour ago",
+        }""",
+                query="""The given html is the crawled html from Google search result. Please find the schema for Top Story item int he given html, I am interested in title, link, source. date and imageUrl."""
+            )
+
+            with open(f"{home_dir}/schema/top_stories_schema.json", "w") as f:
+                f.write(json.dumps(top_stories_schema))
+
+        suggested_query_schema = None
+        if os.path.exists(f"{home_dir}/schema/suggested_query_schema.json"):
+            with open(f"{home_dir}/schema/suggested_query_schema.json", "r") as f:
+                suggested_query_schema = json.load(f)
+        else:
+            suggested_query_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "query": "A for Apple",
+        }""",
+                query="""The given HTML contains the crawled HTML from Google search results. Please find the schema for each suggested query in the section "People also search for" within the given HTML. I am interested in the queries only."""
+            )
+            with open(f"{home_dir}/schema/suggested_query_schema.json", "w") as f:
+                f.write(json.dumps(suggested_query_schema))
+
+        return {
+            "organic_schema": organic_schema,
+            "top_stories_schema": top_stories_schema,
+            "suggested_query_schema": suggested_query_schema,
+        }

crawl4ai/crawlers/google_search/script.js

@@ -0,0 +1,115 @@
+(() => {
+    // Function to extract image data from Google Images page
+    function extractImageData() {
+        const keys = Object.keys(window.W_jd);
+        let allImageData = [];
+        let currentPosition = 0;
+
+        // Get the symbol we'll use (from first valid entry)
+        let targetSymbol;
+        for (let key of keys) {
+            try {
+                const symbols = Object.getOwnPropertySymbols(window.W_jd[key]);
+                if (symbols.length > 0) {
+                    targetSymbol = symbols[0];
+                    break;
+                }
+            } catch (e) {
+                continue;
+            }
+        }
+
+        if (!targetSymbol) return [];
+
+        // Iterate through ALL keys
+        for (let key of keys) {
+            try {
+                const o1 = window.W_jd[key][targetSymbol]
+                if (!o1) continue;
+                const data = Object.values(o1)[0]
+                // const data = window.W_jd[key][targetSymbol]?.Ws;
+                // Check if this is a valid image data entry
+                if (data && Array.isArray(data[1])) {
+                    const processedData = processImageEntry(data, currentPosition);
+                    if (processedData) {
+                        allImageData.push(processedData);
+                        currentPosition++;
+                    }
+                }
+            } catch (e) {
+                continue;
+            }
+        }
+
+        return allImageData;
+    }
+
+    function processImageEntry(entry, position) {
+        const imageData = entry[1];
+        if (!Array.isArray(imageData)) return null;
+
+        // Extract the image ID
+        const imageId = imageData[1];
+        if (!imageId) return null;
+
+        // Find the corresponding DOM element
+        const domElement = document.querySelector(`[data-docid="${imageId}"]`);
+        if (!domElement) return null;
+
+        // Extract data from the array structure
+        const [
+            _,
+            id,
+            thumbnailInfo,
+            imageInfo,
+            __,
+            ___,
+            rgb,
+            ____,
+            _____,
+            metadata
+        ] = imageData;
+
+        // Ensure we have the required data
+        if (!thumbnailInfo || !imageInfo) return null;
+
+        // Extract metadata from DOM
+        const title = domElement?.querySelector('.toI8Rb')?.textContent?.trim();
+        const source = domElement?.querySelector('.guK3rf')?.textContent?.trim();
+        const link = domElement?.querySelector('a.EZAeBe')?.href;
+
+        if (!link) return null;
+
+        // Build Google Image URL
+        const googleUrl = buildGoogleImageUrl(imageInfo[0], link, imageId, imageInfo[1], imageInfo[2]);
+
+        return {
+            title,
+            imageUrl: imageInfo[0],
+            imageWidth: imageInfo[2],
+            imageHeight: imageInfo[1],
+            thumbnailUrl: thumbnailInfo[0],
+            thumbnailWidth: thumbnailInfo[2],
+            thumbnailHeight: thumbnailInfo[1],
+            source,
+            domain: metadata['2000']?.[1] || new URL(link).hostname,
+            link,
+            googleUrl,
+            position: position + 1
+        };
+    }
+
+    function buildGoogleImageUrl(imgUrl, refUrl, tbnid, height, width) {
+        const params = new URLSearchParams({
+            imgurl: imgUrl,
+            tbnid: tbnid,
+            imgrefurl: refUrl,
+            docid: tbnid,
+            w: width.toString(),
+            h: height.toString(),
+        });
+
+        return `https://www.google.com/imgres?${params.toString()}`;
+    }
+    return extractImageData();
+})();

crawl4ai/deep_crawling/__init__.py

@@ -0,0 +1,47 @@
+# deep_crawling/__init__.py
+from .base_strategy import DeepCrawlDecorator, DeepCrawlStrategy
+from .bfs_strategy import BFSDeepCrawlStrategy
+from .bff_strategy import BestFirstCrawlingStrategy
+from .dfs_strategy import DFSDeepCrawlStrategy
+from .filters import (
+    FilterChain,
+    ContentTypeFilter,
+    DomainFilter,
+    URLFilter,
+    URLPatternFilter,
+    FilterStats,
+    ContentRelevanceFilter,
+    SEOFilter
+)
+from .scorers import (
+    KeywordRelevanceScorer,
+    URLScorer,
+    CompositeScorer,
+    DomainAuthorityScorer,
+    FreshnessScorer,
+    PathDepthScorer,
+    ContentTypeScorer
+)
+
+__all__ = [
+    "DeepCrawlDecorator",
+    "DeepCrawlStrategy",
+    "BFSDeepCrawlStrategy",
+    "BestFirstCrawlingStrategy",
+    "DFSDeepCrawlStrategy",
+    "FilterChain",
+    "ContentTypeFilter",
+    "DomainFilter",
+    "URLFilter",
+    "URLPatternFilter",
+    "FilterStats",
+    "ContentRelevanceFilter",
+    "SEOFilter",
+    "KeywordRelevanceScorer",
+    "URLScorer",
+    "CompositeScorer",
+    "DomainAuthorityScorer",
+    "FreshnessScorer",
+    "PathDepthScorer",
+    "ContentTypeScorer",
+]

crawl4ai/deep_crawling/base_strategy.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator, Optional, Set, List, Dict
+from functools import wraps
+from contextvars import ContextVar
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+
+
+class DeepCrawlDecorator:
+    """Decorator that adds deep crawling capability to arun method."""
+    deep_crawl_active = ContextVar("deep_crawl_active", default=False)
+    
+    def __init__(self, crawler: AsyncWebCrawler): 
+        self.crawler = crawler
+
+    def __call__(self, original_arun):
+        @wraps(original_arun)
+        async def wrapped_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
+            # If deep crawling is already active, call the original method to avoid recursion.
+            if config and config.deep_crawl_strategy and not self.deep_crawl_active.get():
+                token = self.deep_crawl_active.set(True)
+                # Await the arun call to get the actual result object.
+                result_obj = await config.deep_crawl_strategy.arun(
+                    crawler=self.crawler,
+                    start_url=url,
+                    config=config
+                )
+                if config.stream:
+                    async def result_wrapper():
+                        try:
+                            async for result in result_obj:
+                                yield result
+                        finally:
+                            self.deep_crawl_active.reset(token)
+                    return result_wrapper()
+                else:
+                    try:
+                        return result_obj
+                    finally:
+                        self.deep_crawl_active.reset(token)
+            return await original_arun(url, config=config, **kwargs)
+        return wrapped_arun
+
+class DeepCrawlStrategy(ABC):
+    """
+    Abstract base class for deep crawling strategies.
+    
+    Core functions:
+      - arun: Main entry point that returns an async generator of CrawlResults.
+      - shutdown: Clean up resources.
+      - can_process_url: Validate a URL and decide whether to process it.
+      - _process_links: Extract and process links from a CrawlResult.
+    """
+
+    @abstractmethod
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) mode:
+        Processes one BFS level at a time, then yields all the results.
+        """
+        pass
+
+    @abstractmethod
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming mode:
+        Processes one BFS level at a time and yields results immediately as they arrive.
+        """
+        pass
+    
+    async def arun(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: Optional[CrawlerRunConfig] = None,
+    ) -> RunManyReturn:
+        """
+        Traverse the given URL using the specified crawler.
+        
+        Args:
+            start_url (str): The URL from which to start crawling.
+            crawler (AsyncWebCrawler): The crawler instance to use.
+            crawler_run_config (Optional[CrawlerRunConfig]): Crawler configuration.
+        
+        Returns:
+            Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+        """
+        if config is None:
+            raise ValueError("CrawlerRunConfig must be provided")
+
+        if config.stream:
+            return self._arun_stream(start_url, crawler, config)
+        else:
+            return await self._arun_batch(start_url, crawler, config)
+
+    def __call__(self, start_url: str, crawler: AsyncWebCrawler, config: CrawlerRunConfig):
+        return self.arun(start_url, crawler, config)
+
+    @abstractmethod
+    async def shutdown(self) -> None:
+        """
+        Clean up resources used by the deep crawl strategy.
+        """
+        pass
+
+    @abstractmethod
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validate the URL format and apply custom filtering logic.
+        
+        Args:
+            url (str): The URL to validate.
+            depth (int): The current depth in the crawl.
+        
+        Returns:
+            bool: True if the URL should be processed, False otherwise.
+        """
+        pass
+
+    @abstractmethod
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_level: List[tuple],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extract and process links from the given crawl result.
+        
+        This method should:
+          - Validate each extracted URL using can_process_url.
+          - Optionally score URLs.
+          - Append valid URLs (and their parent references) to the next_level list.
+          - Update the depths dictionary with the new depth for each URL.
+        
+        Args:
+            result (CrawlResult): The result from a crawl operation.
+            source_url (str): The URL from which this result was obtained.
+            current_depth (int): The depth at which the source URL was processed.
+            visited (Set[str]): Set of already visited URLs.
+            next_level (List[tuple]): List of tuples (url, parent_url) for the next BFS level.
+            depths (Dict[str, int]): Mapping of URLs to their current depth.
+        """
+        pass
+

crawl4ai/deep_crawling/bff_strategy.py

@@ -0,0 +1,255 @@
+# best_first_crawling_strategy.py
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+from urllib.parse import urlparse
+
+from ..models import TraversalStats
+from .filters import FilterChain
+from .scorers import URLScorer
+from . import DeepCrawlStrategy
+
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+
+from math import inf as infinity
+
+# Configurable batch size for processing items from the priority queue
+BATCH_SIZE = 10
+
+
+class BestFirstCrawlingStrategy(DeepCrawlStrategy):
+    """
+    Best-First Crawling Strategy using a priority queue.
+    
+    This strategy prioritizes URLs based on their score, ensuring that higher-value
+    pages are crawled first. It reimplements the core traversal loop to use a priority
+    queue while keeping URL validation and link discovery consistent with our design.
+    
+    Core methods:
+      - arun: Returns either a list (batch mode) or an async generator (stream mode).
+      - _arun_best_first: Core generator that uses a priority queue to yield CrawlResults.
+      - can_process_url: Validates URLs and applies filtering (inherited behavior).
+      - link_discovery: Extracts and validates links from a CrawlResult.
+    """
+    def __init__(
+        self,
+        max_depth: int,
+        filter_chain: FilterChain = FilterChain(),
+        url_scorer: Optional[URLScorer] = None,
+        include_external: bool = False,
+        max_pages: int = infinity,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.url_scorer = url_scorer
+        self.include_external = include_external
+        self.max_pages = max_pages
+        self.logger = logger or logging.getLogger(__name__)
+        self.stats = TraversalStats(start_time=datetime.now())
+        self._cancel_event = asyncio.Event()
+        self._pages_crawled = 0
+
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validate the URL format and apply filtering.
+        For the starting URL (depth 0), filtering is bypassed.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError("Missing scheme or netloc")
+            if parsed.scheme not in ("http", "https"):
+                raise ValueError("Invalid scheme")
+            if "." not in parsed.netloc:
+                raise ValueError("Invalid domain")
+        except Exception as e:
+            self.logger.warning(f"Invalid URL: {url}, error: {e}")
+            return False
+
+        if depth != 0 and not await self.filter_chain.apply(url):
+            return False
+
+        return True
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_links: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extract links from the crawl result, validate them, and append new URLs
+        (with their parent references) to next_links.
+        Also updates the depths dictionary.
+        """
+        new_depth = current_depth + 1
+        if new_depth > self.max_depth:
+            return
+            
+        # If we've reached the max pages limit, don't discover new links
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            return
+
+        # Retrieve internal links; include external links if enabled.
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        # If we have more links than remaining capacity, limit how many we'll process
+        valid_links = []
+        for link in links:
+            url = link.get("href")
+            if url in visited:
+                continue
+            if not await self.can_process_url(url, new_depth):
+                self.stats.urls_skipped += 1
+                continue
+                
+            valid_links.append(url)
+            
+        # If we have more valid links than capacity, limit them
+        if len(valid_links) > remaining_capacity:
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
+            
+        # Record the new depths and add to next_links
+        for url in valid_links:
+            depths[url] = new_depth
+            next_links.append((url, source_url))
+
+    async def _arun_best_first(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Core best-first crawl method using a priority queue.
+        
+        The queue items are tuples of (score, depth, url, parent_url). Lower scores
+        are treated as higher priority. URLs are processed in batches for efficiency.
+        """
+        queue: asyncio.PriorityQueue = asyncio.PriorityQueue()
+        # Push the initial URL with score 0 and depth 0.
+        await queue.put((0, 0, start_url, None))
+        visited: Set[str] = set()
+        depths: Dict[str, int] = {start_url: 0}
+
+        while not queue.empty() and not self._cancel_event.is_set():
+            # Stop if we've reached the max pages limit
+            if self._pages_crawled >= self.max_pages:
+                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                break
+                
+            batch: List[Tuple[float, int, str, Optional[str]]] = []
+            # Retrieve up to BATCH_SIZE items from the priority queue.
+            for _ in range(BATCH_SIZE):
+                if queue.empty():
+                    break
+                item = await queue.get()
+                score, depth, url, parent_url = item
+                if url in visited:
+                    continue
+                visited.add(url)
+                batch.append(item)
+
+            if not batch:
+                continue
+
+            # Process the current batch of URLs.
+            urls = [item[2] for item in batch]
+            batch_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=urls, config=batch_config)
+            async for result in stream_gen:
+                result_url = result.url
+                # Find the corresponding tuple from the batch.
+                corresponding = next((item for item in batch if item[2] == result_url), None)
+                if not corresponding:
+                    continue
+                score, depth, url, parent_url = corresponding
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent_url
+                result.metadata["score"] = score
+                
+                # Count only successful crawls toward max_pages limit
+                if result.success:
+                    self._pages_crawled += 1
+                
+                yield result
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Discover new links from this result
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, result_url, depth, visited, new_links, depths)
+                    
+                    for new_url, new_parent in new_links:
+                        new_depth = depths.get(new_url, depth + 1)
+                        new_score = self.url_scorer.score(new_url) if self.url_scorer else 0
+                        await queue.put((new_score, new_depth, new_url, new_parent))
+
+        # End of crawl.
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Best-first crawl in batch mode.
+        
+        Aggregates all CrawlResults into a list.
+        """
+        results: List[CrawlResult] = []
+        async for result in self._arun_best_first(start_url, crawler, config):
+            results.append(result)
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Best-first crawl in streaming mode.
+        
+        Yields CrawlResults as they become available.
+        """
+        async for result in self._arun_best_first(start_url, crawler, config):
+            yield result
+
+    async def arun(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: Optional[CrawlerRunConfig] = None,
+    ) -> "RunManyReturn":
+        """
+        Main entry point for best-first crawling.
+        
+        Returns either a list (batch mode) or an async generator (stream mode)
+        of CrawlResults.
+        """
+        if config is None:
+            raise ValueError("CrawlerRunConfig must be provided")
+        if config.stream:
+            return self._arun_stream(start_url, crawler, config)
+        else:
+            return await self._arun_batch(start_url, crawler, config)
+
+    async def shutdown(self) -> None:
+        """
+        Signal cancellation and clean up resources.
+        """
+        self._cancel_event.set()
+        self.stats.end_time = datetime.now()

crawl4ai/deep_crawling/bfs_strategy.py

@@ -0,0 +1,245 @@
+# bfs_deep_crawl_strategy.py
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+from urllib.parse import urlparse
+
+from ..models import TraversalStats
+from .filters import FilterChain
+from .scorers import URLScorer
+from . import DeepCrawlStrategy  
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
+from ..utils import normalize_url_for_deep_crawl, efficient_normalize_url_for_deep_crawl
+from math import inf as infinity
+
+class BFSDeepCrawlStrategy(DeepCrawlStrategy):
+    """
+    Breadth-First Search deep crawling strategy.
+    
+    Core functions:
+      - arun: Main entry point; splits execution into batch or stream modes.
+      - link_discovery: Extracts, filters, and (if needed) scores the outgoing URLs.
+      - can_process_url: Validates URL format and applies the filter chain.
+    """
+    def __init__(
+        self,
+        max_depth: int,
+        filter_chain: FilterChain = FilterChain(),
+        url_scorer: Optional[URLScorer] = None,        
+        include_external: bool = False,
+        score_threshold: float = -infinity,
+        max_pages: int = infinity,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.url_scorer = url_scorer
+        self.include_external = include_external
+        self.score_threshold = score_threshold
+        self.max_pages = max_pages
+        self.logger = logger or logging.getLogger(__name__)
+        self.stats = TraversalStats(start_time=datetime.now())
+        self._cancel_event = asyncio.Event()
+        self._pages_crawled = 0
+
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validates the URL and applies the filter chain.
+        For the start URL (depth 0) filtering is bypassed.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError("Missing scheme or netloc")
+            if parsed.scheme not in ("http", "https"):
+                raise ValueError("Invalid scheme")
+            if "." not in parsed.netloc:
+                raise ValueError("Invalid domain")
+        except Exception as e:
+            self.logger.warning(f"Invalid URL: {url}, error: {e}")
+            return False
+
+        if depth != 0 and not await self.filter_chain.apply(url):
+            return False
+
+        return True
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_level: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extracts links from the crawl result, validates and scores them, and
+        prepares the next level of URLs.
+        Each valid URL is appended to next_level as a tuple (url, parent_url)
+        and its depth is tracked.
+        """            
+        next_depth = current_depth + 1
+        if next_depth > self.max_depth:
+            return
+
+        # If we've reached the max pages limit, don't discover new links
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            return
+
+        # Get internal links and, if enabled, external links.
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        valid_links = []
+        
+        # First collect all valid links
+        for link in links:
+            url = link.get("href")
+            # Strip URL fragments to avoid duplicate crawling
+            # base_url = url.split('#')[0] if url else url
+            base_url = normalize_url_for_deep_crawl(url, source_url)
+            if base_url in visited:
+                continue
+            if not await self.can_process_url(url, next_depth):
+                self.stats.urls_skipped += 1
+                continue
+
+            # Score the URL if a scorer is provided
+            score = self.url_scorer.score(base_url) if self.url_scorer else 0
+            
+            # Skip URLs with scores below the threshold
+            if score < self.score_threshold:
+                self.logger.debug(f"URL {url} skipped: score {score} below threshold {self.score_threshold}")
+                self.stats.urls_skipped += 1
+                continue
+            
+            valid_links.append((base_url, score))
+        
+        # If we have more valid links than capacity, sort by score and take the top ones
+        if len(valid_links) > remaining_capacity:
+            if self.url_scorer:
+                # Sort by score in descending order
+                valid_links.sort(key=lambda x: x[1], reverse=True)
+            # Take only as many as we have capacity for
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
+            
+        # Process the final selected links
+        for url, score in valid_links:
+            # attach the score to metadata if needed
+            if score:
+                result.metadata = result.metadata or {}
+                result.metadata["score"] = score
+            next_level.append((url, source_url))
+            depths[url] = next_depth
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) mode:
+        Processes one BFS level at a time, then yields all the results.
+        """
+        visited: Set[str] = set()
+        # current_level holds tuples: (url, parent_url)
+        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        results: List[CrawlResult] = []
+
+        while current_level and not self._cancel_event.is_set():
+            next_level: List[Tuple[str, Optional[str]]] = []
+            urls = [url for url, _ in current_level]
+            visited.update(urls)
+
+            # Clone the config to disable deep crawling recursion and enforce batch mode.
+            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
+            batch_results = await crawler.arun_many(urls=urls, config=batch_config)
+            
+            # Update pages crawled counter - count only successful crawls
+            successful_results = [r for r in batch_results if r.success]
+            self._pages_crawled += len(successful_results)
+            
+            for result in batch_results:
+                url = result.url
+                depth = depths.get(url, 0)
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                result.metadata["parent_url"] = parent_url
+                results.append(result)
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Link discovery will handle the max pages limit internally
+                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+
+            current_level = next_level
+
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming mode:
+        Processes one BFS level at a time and yields results immediately as they arrive.
+        """
+        visited: Set[str] = set()
+        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        while current_level and not self._cancel_event.is_set():
+            next_level: List[Tuple[str, Optional[str]]] = []
+            urls = [url for url, _ in current_level]
+            visited.update(urls)
+
+            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=urls, config=stream_config)
+            
+            # Keep track of processed results for this batch
+            results_count = 0
+            async for result in stream_gen:
+                url = result.url
+                depth = depths.get(url, 0)
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                result.metadata["parent_url"] = parent_url
+                
+                # Count only successful crawls
+                if result.success:
+                    self._pages_crawled += 1
+                
+                results_count += 1
+                yield result
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Link discovery will handle the max pages limit internally
+                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+            
+            # If we didn't get results back (e.g. due to errors), avoid getting stuck in an infinite loop
+            # by considering these URLs as visited but not counting them toward the max_pages limit
+            if results_count == 0 and urls:
+                self.logger.warning(f"No results returned for {len(urls)} URLs, marking as visited")
+                
+            current_level = next_level
+
+    async def shutdown(self) -> None:
+        """
+        Clean up resources and signal cancellation of the crawl.
+        """
+        self._cancel_event.set()
+        self.stats.end_time = datetime.now()

crawl4ai/deep_crawling/crazy.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+# I just got crazy, trying to wrute K&R C but in Python. Right now I feel like I'm in a quantum state.
+# I probably won't use this; I just want to leave it here. A century later, the future human race will be like, "WTF?"
+
+# ------ Imports That Will Make You Question Reality ------ #
+from functools import wraps
+from contextvars import ContextVar
+import inspect
+
+from crawl4ai import CacheMode
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai.models import CrawlResult, TraversalStats
+from crawl4ai.deep_crawling.filters import FilterChain
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+import time
+import logging
+from urllib.parse import urlparse
+
+from abc import ABC, abstractmethod
+from collections import deque
+import asyncio
+from typing import (
+    AsyncGenerator,
+    Dict,
+    List,
+    TypeVar,
+    Generic,
+    Tuple,
+    Callable,
+    Awaitable,
+    Union,
+)
+from functools import lru_cache
+import mmh3
+from bitarray import bitarray
+import numpy as np
+from heapq import heappush, heappop
+
+# ------ Type Algebra Mastery ------ #
+CrawlResultT = TypeVar("CrawlResultT", bound="CrawlResult")
+PriorityT = TypeVar("PriorityT")
+P = TypeVar("P")
+
+# ------ Hyperscalar Context Management ------ #
+deep_crawl_ctx = ContextVar("deep_crawl_stack", default=deque())
+
+# ------ Algebraic Crawler Monoid ------ #
+class TraversalContext:
+    __slots__ = ('visited', 'frontier', 'depths', 'priority_fn', 'current_depth')
+    
+    def __init__(self,
+                 priority_fn: Callable[[str], Awaitable[float]] = lambda _: 1.0):
+        self.visited: BloomFilter = BloomFilter(10**6, 0.01)  # 1M items, 1% FP
+        self.frontier: PriorityQueue = PriorityQueue()
+        self.depths: Dict[str, int] = {}
+        self.priority_fn = priority_fn
+        self.current_depth = 0
+
+    def clone_for_level(self) -> TraversalContext:
+        """Monadic context propagation"""
+        new_ctx = TraversalContext(self.priority_fn)
+        new_ctx.visited = self.visited.copy()
+        new_ctx.depths = self.depths.copy()
+        new_ctx.current_depth = self.current_depth
+        return new_ctx
+
+class PriorityQueue(Generic[PriorityT]):
+    """Fibonacci heap-inspired priority queue with O(1) amortized operations"""
+    __slots__ = ('_heap', '_index')
+
+    def __init__(self):
+        self._heap: List[Tuple[PriorityT, float, P]] = []
+        self._index: Dict[P, int] = {}
+
+    def insert(self, priority: PriorityT, item: P) -> None:
+        tiebreaker = time.time()  # Ensure FIFO for equal priorities
+        heappush(self._heap, (priority, tiebreaker, item))
+        self._index[item] = len(self._heap) - 1
+
+    def extract(self, top_n = 1) -> P:
+        items = []
+        for _ in range(top_n):
+            if not self._heap:
+                break
+            priority, _, item = heappop(self._heap)
+            del self._index[item]
+            items.append(item)
+        if not items:
+            raise IndexError("Priority queue empty")
+        return items
+        # while self._heap:
+        #     _, _, item = heappop(self._heap)
+        #     if item in self._index:
+        #         del self._index[item]
+        #         return item
+        raise IndexError("Priority queue empty")
+
+
+    def is_empty(self) -> bool:
+        return not bool(self._heap)
+
+class BloomFilter:
+    """Optimal Bloom filter using murmur3 hash avalanche"""
+    __slots__ = ('size', 'hashes', 'bits')
+
+    def __init__(self, capacity: int, error_rate: float):
+        self.size = self._optimal_size(capacity, error_rate)
+        self.hashes = self._optimal_hashes(capacity, self.size)
+        self.bits = bitarray(self.size)
+        self.bits.setall(False)
+
+    @staticmethod
+    def _optimal_size(n: int, p: float) -> int:
+        m = - (n * np.log(p)) / (np.log(2) ** 2)
+        return int(np.ceil(m))
+
+    @staticmethod
+    def _optimal_hashes(n: int, m: int) -> int:
+        k = (m / n) * np.log(2)
+        return int(np.ceil(k))
+
+    def add(self, item: str) -> None:
+        for seed in range(self.hashes):
+            digest = mmh3.hash(item, seed) % self.size
+            self.bits[digest] = True
+
+    def __contains__(self, item: str) -> bool:
+        return all(
+            self.bits[mmh3.hash(item, seed) % self.size]
+            for seed in range(self.hashes)
+        )
+
+    def copy(self) -> BloomFilter:
+        new = object.__new__(BloomFilter)
+        new.size = self.size
+        new.hashes = self.hashes
+        new.bits = self.bits.copy()
+        return new
+    
+    def __len__(self) -> int:
+        """
+        Estimates the number of items in the filter using the 
+        count of set bits and the formula:
+        n = -m/k * ln(1 - X/m)
+        where:
+            m = size of bit array
+            k = number of hash functions
+            X = count of set bits
+        """
+        set_bits = self.bits.count(True)
+        if set_bits == 0:
+            return 0
+            
+        # Use the inverse bloom filter formula to estimate cardinality
+        return int(
+            -(self.size / self.hashes) * 
+            np.log(1 - set_bits / self.size)
+        )
+    
+    def bit_count(self) -> int:
+        """Returns the raw count of set bits in the filter"""
+        return self.bits.count(True)
+        
+    def __repr__(self) -> str:
+        return f"BloomFilter(est_items={len(self)}, bits={self.bit_count()}/{self.size})"
+
+# ------ Hyper-Optimal Deep Crawl Core ------ #
+class DeepCrawlDecorator:
+    """Metaprogramming marvel: Zero-cost deep crawl abstraction"""
+    def __init__(self, crawler: AsyncWebCrawler):
+        self.crawler = crawler
+
+    def __call__(self, original_arun: Callable) -> Callable:
+        @wraps(original_arun)
+        async def quantum_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
+            stack = deep_crawl_ctx.get()
+            if config and config.deep_crawl_strategy and not stack:
+                stack.append(self.crawler)
+                try:
+                    deep_crawl_ctx.set(stack)
+                    async for result in config.deep_crawl_strategy.traverse(
+                        start_url=url,
+                        crawler=self.crawler,
+                        config=config
+                    ):
+                        yield result
+                finally:
+                    stack.pop()
+                    deep_crawl_ctx.set(stack)
+            else:
+                result = await original_arun(url, config=config, **kwargs)
+                yield result
+        return quantum_arun
+
+
+async def collect_results(url, crawler, config):
+    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
+        setattr(crawler, "arun", getattr(crawler, "original_arun"))
+
+    ret = crawler.arun(url, config=config)
+    # If arun is an async generator, iterate over it
+    if inspect.isasyncgen(ret):
+        return [r async for r in ret]
+    # Otherwise, await the coroutine and normalize to a list
+    result = await ret
+    return result if isinstance(result, list) else [result]
+
+async def collect_many_results(url, crawler, config):
+    # Replace back arun to its original implementation
+    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
+        setattr(crawler, "arun", getattr(crawler, "original_arun"))
+    ret = crawler.arun_many(url, config=config)
+    # If arun is an async generator, iterate over it
+    if inspect.isasyncgen(ret):
+        return [r async for r in ret]
+    # Otherwise, await the coroutine and normalize to a list
+    result = await ret
+    return result if isinstance(result, list) else [result]
+
+
+# ------ Deep Crawl Strategy Interface ------ #
+CrawlResultT = TypeVar("CrawlResultT", bound=CrawlResult)
+# In batch mode we return List[CrawlResult] and in stream mode an AsyncGenerator.
+RunManyReturn = Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+
+
+class DeepCrawlStrategy(ABC):
+    """Abstract base class that will make Dijkstra smile"""
+    @abstractmethod
+    async def traverse(self,
+                      start_url: str,
+                      crawler: AsyncWebCrawler,
+                      config: CrawlerRunConfig) -> RunManyReturn:
+        """Traverse with O(1) memory complexity via generator fusion"""
+        ...
+
+    @abstractmethod
+    def precompute_priority(self, url: str) -> Awaitable[float]:
+        """Quantum-inspired priority precomputation"""
+        pass
+
+    @abstractmethod
+    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
+        """Hilbert-curve optimized link generation"""
+        pass
+
+# ------ BFS That Would Make Knuth Proud ------ #
+
+def calculate_quantum_batch_size(
+    depth: int,
+    max_depth: int,
+    frontier_size: int,
+    visited_size: int
+) -> int:
+    """
+    Calculates optimal batch size for URL processing using quantum-inspired mathematical principles.
+    
+    This function implements a sophisticated batch size calculation using:
+    1. Golden Ratio (φ) based scaling for optimal irrationality
+    2. Depth-aware amplitude modulation
+    3. Harmonic series dampening
+    4. Logarithmic growth control
+    5. Dynamic frontier adaptation
+    
+    The formula follows the quantum harmonic oscillator principle:
+        N = ⌈φ^(2d) * log₂(|V|) * H(d)⁻¹ * min(20, |F|/10)⌉
+    where:
+        φ = Golden Ratio ((1 + √5) / 2)
+        d = depth factor (normalized remaining depth)
+        |V| = size of visited set
+        H(d) = d-th harmonic number
+        |F| = frontier size
+    
+    Args:
+        depth (int): Current traversal depth
+        max_depth (int): Maximum allowed depth
+        frontier_size (int): Current size of frontier queue
+        visited_size (int): Number of URLs visited so far
+    
+    Returns:
+        int: Optimal batch size bounded between 1 and 100
+        
+    Mathematical Properties:
+        - Maintains O(log n) growth with respect to visited size
+        - Provides φ-optimal distribution of resources
+        - Ensures quantum-like state transitions between depths
+        - Harmonically dampened to prevent exponential explosion
+    """
+    # Golden ratio φ = (1 + √5) / 2
+    φ = (1 + 5 ** 0.5) / 2
+    
+    # Calculate normalized depth factor [0, 1]
+    depth_factor = (max_depth - depth) / max_depth if depth < max_depth else 0
+    
+    # Compute harmonic number for current depth
+    harmonic = sum(1/k for k in range(1, depth + 2))
+    
+    # Calculate quantum batch size
+    batch_size = int(np.ceil(
+        (φ ** (depth_factor * 2)) *          # Golden ratio scaling
+        np.log2(visited_size + 2) *          # Logarithmic growth factor
+        (1 / harmonic) *                     # Harmonic dampening
+        max(1, min(20, frontier_size / 10))  # Frontier-aware scaling
+    ))
+    
+    # Enforce practical bounds
+    return max(1, min(100, batch_size))
+
+
+class BFSDeepCrawlStrategy(DeepCrawlStrategy):
+    """Breadth-First Search with Einstein-Rosen bridge optimization"""
+    __slots__ = ('max_depth', 'filter_chain', 'priority_fn', 'stats', '_cancel')
+
+    def __init__(self,
+                 max_depth: int,
+                 filter_chain: FilterChain = FilterChain(),
+                 priority_fn: Callable[[str], Awaitable[float]] = lambda url: 1.0,
+                 logger: logging.Logger = None):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.priority_fn = priority_fn
+        self.stats = TraversalStats()
+        self._cancel = asyncio.Event()
+        self.semaphore = asyncio.Semaphore(1000)
+
+    async def traverse(self,
+                      start_url: str,
+                      crawler: AsyncWebCrawler,
+                      config: CrawlerRunConfig) -> RunManyReturn:
+        """Non-blocking BFS with O(b^d) time complexity awareness"""
+        ctx = TraversalContext(self.priority_fn)
+        ctx.frontier.insert(self.priority_fn(start_url), (start_url, None, 0))
+        ctx.visited.add(start_url)
+        ctx.depths[start_url] = 0
+
+        while not ctx.frontier.is_empty() and not self._cancel.is_set():
+            # Use the best algorith, to find top_n value
+            top_n = calculate_quantum_batch_size(
+                depth=ctx.current_depth,
+                max_depth=self.max_depth,
+                frontier_size=len(ctx.frontier._heap),
+                visited_size=len(ctx.visited)
+            )
+
+            urls = ctx.frontier.extract(top_n=top_n)
+            # url, parent, depth = ctx.frontier.extract(top_n=top_n)
+            if urls:
+                ctx.current_depth = urls[0][2]
+
+            async with self.semaphore:
+                results = await collect_many_results([url for (url, parent, depth) in urls], crawler, config)
+                # results = await asyncio.gather(*[
+                #     collect_results(url, crawler, config) for (url, parent, depth) in urls
+                # ])
+                # result = _result[0]
+                for ix, result in enumerate(results):
+                    url, parent, depth = result.url, urls[ix][1], urls[ix][2]
+                    result.metadata['depth'] = depth
+                    result.metadata['parent'] = parent
+                    yield result
+
+                    if depth < self.max_depth:
+                        async for link in self.link_hypercube(result):
+                            if link not in ctx.visited:
+                                priority = self.priority_fn(link)
+                                ctx.frontier.insert(priority, (link, url, depth + 1))
+                                ctx.visited.add(link)
+                                ctx.depths[link] = depth + 1
+
+    @lru_cache(maxsize=65536)
+    async def validate_url(self, url: str) -> bool:
+        """Memoized URL validation with λ-calculus purity"""
+        try:
+            parsed = urlparse(url)
+            return (parsed.scheme in {'http', 'https'}
+                    and '.' in parsed.netloc
+                    and await self.filter_chain.apply(url))
+        except Exception:
+            return False
+
+    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
+        """Hilbert-ordered link generation with O(1) yield latency"""
+        links = (link['href'] for link in result.links.get('internal', []))
+        validated = filter(self.validate_url, links)
+        for link in sorted(validated, key=lambda x: -self.priority_fn(x)):
+            yield link
+
+    def __aiter__(self) -> AsyncGenerator[CrawlResult, None]:
+        """Native async iterator interface"""
+        return self.traverse()
+
+    async def __anext__(self) -> CrawlResult:
+        """True async iterator protocol implementation"""
+        result = await self.traverse().__anext__()
+        if result:
+            return result
+        raise StopAsyncIteration
+
+    async def precompute_priority(self, url):
+        return super().precompute_priority(url)
+
+    async def shutdown(self):
+        self._cancel.set()
+
+# ------ Usage That Will Drop Jaws ------ #
+async def main():
+    """Quantum crawl example"""
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=2,
+        priority_fn=lambda url: 1.0 / (len(url) + 1e-9),  # Inverse length priority
+        # filter_chain=FilterChain(...)
+    )
+
+    config: CrawlerRunConfig = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        stream=False,
+        verbose=True,
+        cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        run_decorator = DeepCrawlDecorator(crawler)
+        setattr(crawler, "original_arun", crawler.arun)
+        crawler.arun = run_decorator(crawler.arun)
+        start_time = time.perf_counter()
+        async for result in crawler.arun("https://docs.crawl4ai.com", config=config):
+            print(f"🌀 {result.url} (Depth: {result.metadata['depth']})")
+        print(f"Deep crawl completed in {time.perf_counter() - start_time:.2f}s")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

crawl4ai/deep_crawling/dfs_strategy.py

@@ -0,0 +1,102 @@
+# dfs_deep_crawl_strategy.py
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+
+from ..models import CrawlResult
+from .bfs_strategy import BFSDeepCrawlStrategy  # noqa
+from ..types import AsyncWebCrawler, CrawlerRunConfig
+
+class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
+    """
+    Depth-First Search (DFS) deep crawling strategy.
+
+    Inherits URL validation and link discovery from BFSDeepCrawlStrategy.
+    Overrides _arun_batch and _arun_stream to use a stack (LIFO) for DFS traversal.
+    """
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) DFS mode.
+        Uses a stack to traverse URLs in DFS order, aggregating CrawlResults into a list.
+        """
+        visited: Set[str] = set()
+        # Stack items: (url, parent_url, depth)
+        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+        depths: Dict[str, int] = {start_url: 0}
+        results: List[CrawlResult] = []
+
+        while stack and not self._cancel_event.is_set():
+            url, parent, depth = stack.pop()
+            if url in visited or depth > self.max_depth:
+                continue
+            visited.add(url)
+
+            # Clone config to disable recursive deep crawling.
+            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
+            url_results = await crawler.arun_many(urls=[url], config=batch_config)
+            
+            for result in url_results:
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent
+                if self.url_scorer:
+                    result.metadata["score"] = self.url_scorer.score(url)
+                results.append(result)
+                
+                # Count only successful crawls toward max_pages limit
+                if result.success:
+                    self._pages_crawled += 1
+                    
+                    # Only discover links from successful crawls
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    
+                    # Push new links in reverse order so the first discovered is processed next.
+                    for new_url, new_parent in reversed(new_links):
+                        new_depth = depths.get(new_url, depth + 1)
+                        stack.append((new_url, new_parent, new_depth))
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming DFS mode.
+        Uses a stack to traverse URLs in DFS order and yields CrawlResults as they become available.
+        """
+        visited: Set[str] = set()
+        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        while stack and not self._cancel_event.is_set():
+            url, parent, depth = stack.pop()
+            if url in visited or depth > self.max_depth:
+                continue
+            visited.add(url)
+
+            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=[url], config=stream_config)
+            async for result in stream_gen:
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent
+                if self.url_scorer:
+                    result.metadata["score"] = self.url_scorer.score(url)
+                yield result
+
+                # Only count successful crawls toward max_pages limit
+                # and only discover links from successful crawls
+                if result.success:
+                    self._pages_crawled += 1
+                    
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    for new_url, new_parent in reversed(new_links):
+                        new_depth = depths.get(new_url, depth + 1)
+                        stack.append((new_url, new_parent, new_depth))

crawl4ai/deep_crawling/filters.py

@@ -0,0 +1,666 @@
+from abc import ABC, abstractmethod
+from typing import List, Pattern, Set, Union
+from urllib.parse import urlparse
+from array import array
+import re
+import logging
+from functools import lru_cache
+import fnmatch
+from dataclasses import dataclass
+import weakref
+import math
+from collections import defaultdict
+from typing import Dict
+from ..utils import HeadPeekr
+import asyncio
+import inspect
+
+
+@dataclass
+class FilterStats:
+    __slots__ = ("_counters",)
+
+    def __init__(self):
+        # Use array of unsigned ints for atomic operations
+        self._counters = array("I", [0, 0, 0])  # total, passed, rejected
+
+    @property
+    def total_urls(self):
+        return self._counters[0]
+
+    @property
+    def passed_urls(self):
+        return self._counters[1]
+
+    @property
+    def rejected_urls(self):
+        return self._counters[2]
+
+
+class URLFilter(ABC):
+    """Optimized base filter class"""
+
+    __slots__ = ("name", "stats", "_logger_ref")
+
+    def __init__(self, name: str = None):
+        self.name = name or self.__class__.__name__
+        self.stats = FilterStats()
+        # Lazy logger initialization using weakref
+        self._logger_ref = None
+
+    @property
+    def logger(self):
+        if self._logger_ref is None or self._logger_ref() is None:
+            logger = logging.getLogger(f"urlfilter.{self.name}")
+            self._logger_ref = weakref.ref(logger)
+        return self._logger_ref()
+
+    @abstractmethod
+    def apply(self, url: str) -> bool:
+        pass
+
+    def _update_stats(self, passed: bool):
+        # Use direct array index for speed
+        self.stats._counters[0] += 1  # total
+        self.stats._counters[1] += passed  # passed
+        self.stats._counters[2] += not passed  # rejected
+
+
+class FilterChain:
+    """Optimized filter chain"""
+
+    __slots__ = ("filters", "stats", "_logger_ref")
+
+    def __init__(self, filters: List[URLFilter] = None):
+        self.filters = tuple(filters or [])  # Immutable tuple for speed
+        self.stats = FilterStats()
+        self._logger_ref = None
+
+    @property
+    def logger(self):
+        if self._logger_ref is None or self._logger_ref() is None:
+            logger = logging.getLogger("urlfilter.chain")
+            self._logger_ref = weakref.ref(logger)
+        return self._logger_ref()
+
+    def add_filter(self, filter_: URLFilter) -> "FilterChain":
+        """Add a filter to the chain"""
+        self.filters.append(filter_)
+        return self  # Enable method chaining
+
+    async def apply(self, url: str) -> bool:
+        """Apply all filters concurrently when possible"""
+        self.stats._counters[0] += 1  # Total processed URLs
+
+        tasks = []
+        for f in self.filters:
+            result = f.apply(url)
+
+            if inspect.isawaitable(result):
+                tasks.append(result)  # Collect async tasks
+            elif not result:  # Sync rejection
+                self.stats._counters[2] += 1  # Sync rejected
+                return False
+
+        if tasks:
+            results = await asyncio.gather(*tasks)
+
+            # Count how many filters rejected
+            rejections = results.count(False)
+            self.stats._counters[2] += rejections
+
+            if not all(results):
+                return False  # Stop early if any filter rejected
+
+        self.stats._counters[1] += 1  # Passed
+        return True
+
+
+class URLPatternFilter(URLFilter):
+    """Pattern filter balancing speed and completeness"""
+
+    __slots__ = (
+        "_simple_suffixes",
+        "_simple_prefixes",
+        "_domain_patterns",
+        "_path_patterns",
+        "_reverse",
+    )
+
+    PATTERN_TYPES = {
+        "SUFFIX": 1,  # *.html
+        "PREFIX": 2,  # /foo/*
+        "DOMAIN": 3,  # *.example.com
+        "PATH": 4,  # Everything else
+        "REGEX": 5,
+    }
+
+    def __init__(
+        self,
+        patterns: Union[str, Pattern, List[Union[str, Pattern]]],
+        use_glob: bool = True,
+        reverse: bool = False,
+    ):
+        super().__init__()
+        self._reverse = reverse
+        patterns = [patterns] if isinstance(patterns, (str, Pattern)) else patterns
+
+        self._simple_suffixes = set()
+        self._simple_prefixes = set()
+        self._domain_patterns = []
+        self._path_patterns = []
+
+        for pattern in patterns:
+            pattern_type = self._categorize_pattern(pattern)
+            self._add_pattern(pattern, pattern_type)
+
+    def _categorize_pattern(self, pattern: str) -> int:
+        """Categorize pattern for specialized handling"""
+        if not isinstance(pattern, str):
+            return self.PATTERN_TYPES["PATH"]
+
+        # Check if it's a regex pattern
+        if pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern:
+            return self.PATTERN_TYPES["REGEX"]
+
+        if pattern.count("*") == 1:
+            if pattern.startswith("*."):
+                return self.PATTERN_TYPES["SUFFIX"]
+            if pattern.endswith("/*"):
+                return self.PATTERN_TYPES["PREFIX"]
+
+        if "://" in pattern and pattern.startswith("*."):
+            return self.PATTERN_TYPES["DOMAIN"]
+
+        return self.PATTERN_TYPES["PATH"]
+
+    def _add_pattern(self, pattern: str, pattern_type: int):
+        """Add pattern to appropriate matcher"""
+        if pattern_type == self.PATTERN_TYPES["REGEX"]:
+            # For regex patterns, compile directly without glob translation
+            if isinstance(pattern, str) and (
+                pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern
+            ):
+                self._path_patterns.append(re.compile(pattern))
+                return
+        elif pattern_type == self.PATTERN_TYPES["SUFFIX"]:
+            self._simple_suffixes.add(pattern[2:])
+        elif pattern_type == self.PATTERN_TYPES["PREFIX"]:
+            self._simple_prefixes.add(pattern[:-2])
+        elif pattern_type == self.PATTERN_TYPES["DOMAIN"]:
+            self._domain_patterns.append(re.compile(pattern.replace("*.", r"[^/]+\.")))
+        else:
+            if isinstance(pattern, str):
+                # Handle complex glob patterns
+                if "**" in pattern:
+                    pattern = pattern.replace("**", ".*")
+                if "{" in pattern:
+                    # Convert {a,b} to (a|b)
+                    pattern = re.sub(
+                        r"\{([^}]+)\}",
+                        lambda m: f'({"|".join(m.group(1).split(","))})',
+                        pattern,
+                    )
+                pattern = fnmatch.translate(pattern)
+            self._path_patterns.append(
+                pattern if isinstance(pattern, Pattern) else re.compile(pattern)
+            )
+
+    @lru_cache(maxsize=10000)
+    def apply(self, url: str) -> bool:
+        # Quick suffix check (*.html)
+        if self._simple_suffixes:
+            path = url.split("?")[0]
+            if path.split("/")[-1].split(".")[-1] in self._simple_suffixes:
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
+
+        # Domain check
+        if self._domain_patterns:
+            for pattern in self._domain_patterns:
+                if pattern.match(url):
+                    result = True
+                    self._update_stats(result)
+                    return not result if self._reverse else result
+
+        # Prefix check (/foo/*)
+        if self._simple_prefixes:
+            path = url.split("?")[0]
+            if any(path.startswith(p) for p in self._simple_prefixes):
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
+
+        # Complex patterns
+        if self._path_patterns:
+            if any(p.search(url) for p in self._path_patterns):
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
+
+        result = False
+        self._update_stats(result)
+        return not result if self._reverse else result
+
+
+class ContentTypeFilter(URLFilter):
+    """Optimized content type filter using fast lookups"""
+
+    __slots__ = ("allowed_types", "_ext_map", "_check_extension")
+
+    # Fast extension to mime type mapping
+    _MIME_MAP = {
+        # Text Formats
+        "txt": "text/plain",
+        "html": "text/html",
+        "htm": "text/html",
+        "xhtml": "application/xhtml+xml",
+        "css": "text/css",
+        "csv": "text/csv",
+        "ics": "text/calendar",
+        "js": "application/javascript",
+        # Images
+        "bmp": "image/bmp",
+        "gif": "image/gif",
+        "jpeg": "image/jpeg",
+        "jpg": "image/jpeg",
+        "png": "image/png",
+        "svg": "image/svg+xml",
+        "tiff": "image/tiff",
+        "ico": "image/x-icon",
+        "webp": "image/webp",
+        # Audio
+        "mp3": "audio/mpeg",
+        "wav": "audio/wav",
+        "ogg": "audio/ogg",
+        "m4a": "audio/mp4",
+        "aac": "audio/aac",
+        # Video
+        "mp4": "video/mp4",
+        "mpeg": "video/mpeg",
+        "webm": "video/webm",
+        "avi": "video/x-msvideo",
+        "mov": "video/quicktime",
+        "flv": "video/x-flv",
+        "wmv": "video/x-ms-wmv",
+        "mkv": "video/x-matroska",
+        # Applications
+        "json": "application/json",
+        "xml": "application/xml",
+        "pdf": "application/pdf",
+        "zip": "application/zip",
+        "gz": "application/gzip",
+        "tar": "application/x-tar",
+        "rar": "application/vnd.rar",
+        "7z": "application/x-7z-compressed",
+        "exe": "application/vnd.microsoft.portable-executable",
+        "msi": "application/x-msdownload",
+        # Fonts
+        "woff": "font/woff",
+        "woff2": "font/woff2",
+        "ttf": "font/ttf",
+        "otf": "font/otf",
+        # Microsoft Office
+        "doc": "application/msword",
+        "dot": "application/msword",
+        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "xls": "application/vnd.ms-excel",
+        "ppt": "application/vnd.ms-powerpoint",
+        "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        # OpenDocument Formats
+        "odt": "application/vnd.oasis.opendocument.text",
+        "ods": "application/vnd.oasis.opendocument.spreadsheet",
+        "odp": "application/vnd.oasis.opendocument.presentation",
+        # Archives
+        "tar.gz": "application/gzip",
+        "tgz": "application/gzip",
+        "bz2": "application/x-bzip2",
+        # Others
+        "rtf": "application/rtf",
+        "apk": "application/vnd.android.package-archive",
+        "epub": "application/epub+zip",
+        "jar": "application/java-archive",
+        "swf": "application/x-shockwave-flash",
+        "midi": "audio/midi",
+        "mid": "audio/midi",
+        "ps": "application/postscript",
+        "ai": "application/postscript",
+        "eps": "application/postscript",
+        # Custom or less common
+        "bin": "application/octet-stream",
+        "dmg": "application/x-apple-diskimage",
+        "iso": "application/x-iso9660-image",
+        "deb": "application/x-debian-package",
+        "rpm": "application/x-rpm",
+        "sqlite": "application/vnd.sqlite3",
+        # Placeholder
+        "unknown": "application/octet-stream",  # Fallback for unknown file types
+    }
+
+    @staticmethod
+    @lru_cache(maxsize=1000)
+    def _extract_extension(url: str) -> str:
+        """Extracts file extension from a URL."""
+        # Remove scheme (http://, https://) if present
+        if "://" in url:
+            url = url.split("://", 1)[-1]  # Get everything after '://'
+
+        # Remove domain (everything up to the first '/')
+        path_start = url.find("/")
+        path = url[path_start:] if path_start != -1 else ""
+
+        # Extract last filename in path
+        filename = path.rsplit("/", 1)[-1] if "/" in path else ""
+
+        # Extract and validate extension
+        if "." not in filename:
+            return ""
+
+        return filename.rpartition(".")[-1].lower()
+
+    def __init__(
+        self,
+        allowed_types: Union[str, List[str]],
+        check_extension: bool = True,
+        ext_map: Dict[str, str] = _MIME_MAP,
+    ):
+        super().__init__()
+        # Normalize and store as frozenset for fast lookup
+        self.allowed_types = frozenset(
+            t.lower()
+            for t in (
+                allowed_types if isinstance(allowed_types, list) else [allowed_types]
+            )
+        )
+        self._check_extension = check_extension
+
+        # Pre-compute extension map for allowed types
+        self._ext_map = frozenset(
+            ext
+            for ext, mime in self._MIME_MAP.items()
+            if any(allowed in mime for allowed in self.allowed_types)
+        )
+
+    @lru_cache(maxsize=1000)
+    def _check_url_cached(self, url: str) -> bool:
+        """Cached URL checking"""
+        if not self._check_extension:
+            return True
+        ext = self._extract_extension(url)
+        if not ext:
+            return True
+
+        return ext in self._ext_map
+
+    def apply(self, url: str) -> bool:
+        """Fast extension check with caching"""
+        result = self._check_url_cached(url)
+        self._update_stats(result)
+        return result
+
+
+class DomainFilter(URLFilter):
+    """Optimized domain filter with fast lookups and caching"""
+
+    __slots__ = ("_allowed_domains", "_blocked_domains", "_domain_cache")
+
+    # Regex for fast domain extraction
+    _DOMAIN_REGEX = re.compile(r"://([^/]+)")
+
+    def __init__(
+        self,
+        allowed_domains: Union[str, List[str]] = None,
+        blocked_domains: Union[str, List[str]] = None,
+    ):
+        super().__init__()
+
+        # Convert inputs to frozensets for immutable, fast lookups
+        self._allowed_domains = (
+            frozenset(self._normalize_domains(allowed_domains))
+            if allowed_domains
+            else None
+        )
+        self._blocked_domains = (
+            frozenset(self._normalize_domains(blocked_domains))
+            if blocked_domains
+            else frozenset()
+        )
+
+    @staticmethod
+    def _normalize_domains(domains: Union[str, List[str]]) -> Set[str]:
+        """Fast domain normalization"""
+        if isinstance(domains, str):
+            return {domains.lower()}
+        return {d.lower() for d in domains}
+    
+    @staticmethod
+    def _is_subdomain(domain: str, parent_domain: str) -> bool:
+        """Check if domain is a subdomain of parent_domain"""
+        return domain == parent_domain or domain.endswith(f".{parent_domain}")
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _extract_domain(url: str) -> str:
+        """Ultra-fast domain extraction with regex and caching"""
+        match = DomainFilter._DOMAIN_REGEX.search(url)
+        return match.group(1).lower() if match else ""
+
+    def apply(self, url: str) -> bool:
+        """Optimized domain checking with early returns"""
+        # Skip processing if no filters
+        if not self._blocked_domains and self._allowed_domains is None:
+            self._update_stats(True)
+            return True
+
+        domain = self._extract_domain(url)
+
+        # Check for blocked domains, including subdomains
+        for blocked in self._blocked_domains:
+            if self._is_subdomain(domain, blocked):
+                self._update_stats(False)
+                return False
+
+        # If no allowed domains specified, accept all non-blocked
+        if self._allowed_domains is None:
+            self._update_stats(True)
+            return True
+
+        # Check if domain matches any allowed domain (including subdomains)
+        for allowed in self._allowed_domains:
+            if self._is_subdomain(domain, allowed):
+                self._update_stats(True)
+                return True
+
+        # No matches found
+        self._update_stats(False)
+        return False
+
+
+class ContentRelevanceFilter(URLFilter):
+    """BM25-based relevance filter using head section content"""
+
+    __slots__ = ("query_terms", "threshold", "k1", "b", "avgdl")
+
+    def __init__(
+        self,
+        query: str,
+        threshold: float,
+        k1: float = 1.2,
+        b: float = 0.75,
+        avgdl: int = 1000,
+    ):
+        super().__init__(name="BM25RelevanceFilter")
+        self.query_terms = self._tokenize(query)
+        self.threshold = threshold
+        self.k1 = k1  # TF saturation parameter
+        self.b = b  # Length normalization parameter
+        self.avgdl = avgdl  # Average document length (empirical value)
+
+    async def apply(self, url: str) -> bool:
+        head_content = await HeadPeekr.peek_html(url)
+        if not head_content:
+            self._update_stats(False)
+            return False
+
+        # Field extraction with weighting
+        fields = {
+            "title": HeadPeekr.get_title(head_content) or "",
+            "meta": HeadPeekr.extract_meta_tags(head_content),
+        }
+        doc_text = self._build_document(fields)
+
+        score = self._bm25(doc_text)
+        decision = score >= self.threshold
+        self._update_stats(decision)
+        return decision
+
+    def _build_document(self, fields: Dict) -> str:
+        """Weighted document construction"""
+        return " ".join(
+            [
+                fields["title"] * 3,  # Title weight
+                fields["meta"].get("description", "") * 2,
+                fields["meta"].get("keywords", ""),
+                " ".join(fields["meta"].values()),
+            ]
+        )
+
+    def _tokenize(self, text: str) -> List[str]:
+        """Fast case-insensitive tokenization"""
+        return text.lower().split()
+
+    def _bm25(self, document: str) -> float:
+        """Optimized BM25 implementation for head sections"""
+        doc_terms = self._tokenize(document)
+        doc_len = len(doc_terms)
+        tf = defaultdict(int)
+
+        for term in doc_terms:
+            tf[term] += 1
+
+        score = 0.0
+        for term in set(self.query_terms):
+            term_freq = tf[term]
+            idf = math.log((1 + 1) / (term_freq + 0.5) + 1)  # Simplified IDF
+            numerator = term_freq * (self.k1 + 1)
+            denominator = term_freq + self.k1 * (
+                1 - self.b + self.b * (doc_len / self.avgdl)
+            )
+            score += idf * (numerator / denominator)
+
+        return score
+
+
+class SEOFilter(URLFilter):
+    """Quantitative SEO quality assessment filter using head section analysis"""
+
+    __slots__ = ("threshold", "_weights", "_kw_patterns")
+
+    # Based on SEMrush/Google ranking factors research
+    DEFAULT_WEIGHTS = {
+        "title_length": 0.15,
+        "title_kw": 0.18,
+        "meta_description": 0.12,
+        "canonical": 0.10,
+        "robot_ok": 0.20,  # Most critical factor
+        "schema_org": 0.10,
+        "url_quality": 0.15,
+    }
+
+    def __init__(
+        self,
+        threshold: float = 0.65,
+        keywords: List[str] = None,
+        weights: Dict[str, float] = None,
+    ):
+        super().__init__(name="SEOFilter")
+        self.threshold = threshold
+        self._weights = weights or self.DEFAULT_WEIGHTS
+        self._kw_patterns = (
+            re.compile(
+                r"\b({})\b".format("|".join(map(re.escape, keywords or []))), re.I
+            )
+            if keywords
+            else None
+        )
+
+    async def apply(self, url: str) -> bool:
+        head_content = await HeadPeekr.peek_html(url)
+        if not head_content:
+            self._update_stats(False)
+            return False
+
+        meta = HeadPeekr.extract_meta_tags(head_content)
+        title = HeadPeekr.get_title(head_content) or ""
+        parsed_url = urlparse(url)
+
+        scores = {
+            "title_length": self._score_title_length(title),
+            "title_kw": self._score_keyword_presence(title),
+            "meta_description": self._score_meta_description(
+                meta.get("description", "")
+            ),
+            "canonical": self._score_canonical(meta.get("canonical"), url),
+            "robot_ok": 1.0 if "noindex" not in meta.get("robots", "") else 0.0,
+            "schema_org": self._score_schema_org(head_content),
+            "url_quality": self._score_url_quality(parsed_url),
+        }
+
+        total_score = sum(
+            weight * scores[factor] for factor, weight in self._weights.items()
+        )
+
+        decision = total_score >= self.threshold
+        self._update_stats(decision)
+        return decision
+
+    def _score_title_length(self, title: str) -> float:
+        length = len(title)
+        if 50 <= length <= 60:
+            return 1.0
+        if 40 <= length < 50 or 60 < length <= 70:
+            return 0.7
+        return 0.3  # Poor length
+
+    def _score_keyword_presence(self, text: str) -> float:
+        if not self._kw_patterns:
+            return 0.0
+        matches = len(self._kw_patterns.findall(text))
+        return min(matches * 0.3, 1.0)  # Max 3 matches
+
+    def _score_meta_description(self, desc: str) -> float:
+        length = len(desc)
+        if 140 <= length <= 160:
+            return 1.0
+        return 0.5 if 120 <= length <= 200 else 0.2
+
+    def _score_canonical(self, canonical: str, original: str) -> float:
+        if not canonical:
+            return 0.5  # Neutral score
+        return 1.0 if canonical == original else 0.2
+
+    def _score_schema_org(self, html: str) -> float:
+        # Detect any schema.org markup in head
+        return (
+            1.0
+            if re.search(r'<script[^>]+type=["\']application/ld\+json', html)
+            else 0.0
+        )
+
+    def _score_url_quality(self, parsed_url) -> float:
+        score = 1.0
+        path = parsed_url.path.lower()
+
+        # Penalty factors
+        if len(path) > 80:
+            score *= 0.7
+        if re.search(r"\d{4}", path):
+            score *= 0.8  # Numbers in path
+        if parsed_url.query:
+            score *= 0.6  # URL parameters
+        if "_" in path:
+            score *= 0.9  # Underscores vs hyphens
+
+        return score

crawl4ai/deep_crawling/scorers.py

@@ -0,0 +1,519 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from urllib.parse import urlparse, unquote
+import re
+import logging
+from functools import lru_cache
+from array import array
+import ctypes
+import platform
+PLATFORM = platform.system()
+
+# Pre-computed scores for common year differences
+_SCORE_LOOKUP = [1.0, 0.5, 0.3333333333333333, 0.25]
+
+# Pre-computed scores for common year differences
+_FRESHNESS_SCORES = [
+   1.0,    # Current year
+   0.9,    # Last year
+   0.8,    # 2 years ago
+   0.7,    # 3 years ago
+   0.6,    # 4 years ago
+   0.5,    # 5 years ago
+]
+
+class ScoringStats:
+    __slots__ = ('_urls_scored', '_total_score', '_min_score', '_max_score')
+    
+    def __init__(self):
+        self._urls_scored = 0
+        self._total_score = 0.0
+        self._min_score = None  # Lazy initialization
+        self._max_score = None
+    
+    def update(self, score: float) -> None:
+        """Optimized update with minimal operations"""
+        self._urls_scored += 1
+        self._total_score += score
+        
+        # Lazy min/max tracking - only if actually accessed
+        if self._min_score is not None:
+            if score < self._min_score:
+                self._min_score = score
+        if self._max_score is not None:
+            if score > self._max_score:
+                self._max_score = score
+                
+    def get_average(self) -> float:
+        """Direct calculation instead of property"""
+        return self._total_score / self._urls_scored if self._urls_scored else 0.0
+    
+    def get_min(self) -> float:
+        """Lazy min calculation"""
+        if self._min_score is None:
+            self._min_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
+        return self._min_score
+        
+    def get_max(self) -> float:
+        """Lazy max calculation"""
+        if self._max_score is None:
+            self._max_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
+        return self._max_score
+class URLScorer(ABC):
+    __slots__ = ('_weight', '_stats')
+    
+    def __init__(self, weight: float = 1.0):
+        # Store weight directly as float32 for memory efficiency
+        self._weight = ctypes.c_float(weight).value
+        self._stats = ScoringStats()
+    
+    @abstractmethod
+    def _calculate_score(self, url: str) -> float:
+        """Calculate raw score for URL."""
+        pass
+    
+    def score(self, url: str) -> float:
+        """Calculate weighted score with minimal overhead."""
+        score = self._calculate_score(url) * self._weight
+        self._stats.update(score)
+        return score
+    
+    @property
+    def stats(self):
+        """Access to scoring statistics."""
+        return self._stats
+    
+    @property
+    def weight(self):
+        return self._weight
+
+class CompositeScorer(URLScorer):
+    __slots__ = ('_scorers', '_normalize', '_weights_array', '_score_array')
+    
+    def __init__(self, scorers: List[URLScorer], normalize: bool = True):
+        """Initialize composite scorer combining multiple scoring strategies.
+        
+        Optimized for:
+        - Fast parallel scoring
+        - Memory efficient score aggregation
+        - Quick short-circuit conditions
+        - Pre-allocated arrays
+        
+        Args:
+            scorers: List of scoring strategies to combine
+            normalize: Whether to normalize final score by scorer count
+        """
+        super().__init__(weight=1.0)
+        self._scorers = scorers
+        self._normalize = normalize
+        
+        # Pre-allocate arrays for scores and weights
+        self._weights_array = array('f', [s.weight for s in scorers])
+        self._score_array = array('f', [0.0] * len(scorers))
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate combined score from all scoring strategies.
+        
+        Uses:
+        1. Pre-allocated arrays for scores
+        2. Short-circuit on zero scores
+        3. Optimized normalization
+        4. Vectorized operations where possible
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Combined and optionally normalized score
+        """
+        total_score = 0.0
+        scores = self._score_array
+        
+        # Get scores from all scorers
+        for i, scorer in enumerate(self._scorers):
+            # Use public score() method which applies weight
+            scores[i] = scorer.score(url)
+            total_score += scores[i]
+            
+        # Normalize if requested
+        if self._normalize and self._scorers:
+            count = len(self._scorers)
+            return total_score / count
+            
+        return total_score
+
+    def score(self, url: str) -> float:
+        """Public scoring interface with stats tracking.
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Final combined score
+        """
+        score = self._calculate_score(url)
+        self.stats.update(score)
+        return score
+
+class KeywordRelevanceScorer(URLScorer):
+    __slots__ = ('_weight', '_stats', '_keywords', '_case_sensitive')
+    
+    def __init__(self, keywords: List[str], weight: float = 1.0, case_sensitive: bool = False):
+        super().__init__(weight=weight)
+        self._case_sensitive = case_sensitive
+        # Pre-process keywords once
+        self._keywords = [k if case_sensitive else k.lower() for k in keywords]
+    
+    @lru_cache(maxsize=10000)
+    def _url_bytes(self, url: str) -> bytes:
+        """Cache decoded URL bytes"""
+        return url.encode('utf-8') if self._case_sensitive else url.lower().encode('utf-8')
+    
+    
+    def _calculate_score(self, url: str) -> float:
+        """Fast string matching without regex or byte conversion"""
+        if not self._case_sensitive:
+            url = url.lower()
+            
+        matches = sum(1 for k in self._keywords if k in url)
+        
+        # Fast return paths
+        if not matches:
+            return 0.0
+        if matches == len(self._keywords):
+            return 1.0
+            
+        return matches / len(self._keywords)
+
+class PathDepthScorer(URLScorer):
+    __slots__ = ('_weight', '_stats', '_optimal_depth')  # Remove _url_cache
+    
+    def __init__(self, optimal_depth: int = 3, weight: float = 1.0):
+        super().__init__(weight=weight)
+        self._optimal_depth = optimal_depth
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _quick_depth(path: str) -> int:
+        """Ultra fast path depth calculation.
+        
+        Examples:
+            - "http://example.com" -> 0  # No path segments
+            - "http://example.com/" -> 0  # Empty path
+            - "http://example.com/a" -> 1
+            - "http://example.com/a/b" -> 2
+        """
+        if not path or path == '/':
+            return 0
+            
+        if '/' not in path:
+            return 0
+            
+        depth = 0
+        last_was_slash = True
+        
+        for c in path:
+            if c == '/':
+                if not last_was_slash:
+                    depth += 1
+                last_was_slash = True
+            else:
+                last_was_slash = False
+                
+        if not last_was_slash:
+            depth += 1
+            
+        return depth
+
+    @lru_cache(maxsize=10000)  # Cache the whole calculation
+    def _calculate_score(self, url: str) -> float:
+        pos = url.find('/', url.find('://') + 3)
+        if pos == -1:
+            depth = 0
+        else:
+            depth = self._quick_depth(url[pos:])
+            
+        # Use lookup table for common distances
+        distance = depth - self._optimal_depth
+        distance = distance if distance >= 0 else -distance  # Faster than abs()
+        
+        if distance < 4:
+            return _SCORE_LOOKUP[distance]
+            
+        return 1.0 / (1.0 + distance)                                             
+
+class ContentTypeScorer(URLScorer):
+    __slots__ = ('_weight', '_exact_types', '_regex_types')
+
+    def __init__(self, type_weights: Dict[str, float], weight: float = 1.0):
+        """Initialize scorer with type weights map.
+        
+        Args:
+            type_weights: Dict mapping file extensions/patterns to scores (e.g. {'.html$': 1.0})
+            weight: Overall weight multiplier for this scorer
+        """
+        super().__init__(weight=weight)
+        self._exact_types = {}  # Fast lookup for simple extensions
+        self._regex_types = []  # Fallback for complex patterns
+        
+        # Split into exact vs regex matchers for performance
+        for pattern, score in type_weights.items():
+            if pattern.startswith('.') and pattern.endswith('$'):
+                ext = pattern[1:-1]
+                self._exact_types[ext] = score
+            else:
+                self._regex_types.append((re.compile(pattern), score))
+                
+        # Sort complex patterns by score for early exit
+        self._regex_types.sort(key=lambda x: -x[1])
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _quick_extension(url: str) -> str:
+        """Extract file extension ultra-fast without regex/splits.
+        
+        Handles:
+        - Basic extensions: "example.html" -> "html"
+        - Query strings: "page.php?id=1" -> "php" 
+        - Fragments: "doc.pdf#page=1" -> "pdf"
+        - Path params: "file.jpg;width=100" -> "jpg"
+        
+        Args:
+            url: URL to extract extension from
+            
+        Returns:
+            Extension without dot, or empty string if none found
+        """
+        pos = url.rfind('.')
+        if pos == -1:
+            return ''
+        
+        # Find first non-alphanumeric char after extension
+        end = len(url)
+        for i in range(pos + 1, len(url)):
+            c = url[i]
+            # Stop at query string, fragment, path param or any non-alphanumeric
+            if c in '?#;' or not c.isalnum():
+                end = i
+                break
+                
+        return url[pos + 1:end].lower()
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate content type score for URL.
+        
+        Uses staged approach:
+        1. Try exact extension match (fast path)
+        2. Fall back to regex patterns if needed
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Score between 0.0 and 1.0 * weight
+        """
+        # Fast path: direct extension lookup
+        ext = self._quick_extension(url)
+        if ext:
+            score = self._exact_types.get(ext, None)
+            if score is not None:
+                return score
+                
+        # Slow path: regex patterns
+        for pattern, score in self._regex_types:
+            if pattern.search(url):
+                return score
+
+        return 0.0
+
+class FreshnessScorer(URLScorer):
+    __slots__ = ('_weight', '_date_pattern', '_current_year')
+
+    def __init__(self, weight: float = 1.0, current_year: int = 2024):
+        """Initialize freshness scorer.
+        
+        Extracts and scores dates from URLs using format:
+        - YYYY/MM/DD 
+        - YYYY-MM-DD
+        - YYYY_MM_DD
+        - YYYY (year only)
+        
+        Args:
+            weight: Score multiplier
+            current_year: Year to calculate freshness against (default 2024)
+        """
+        super().__init__(weight=weight)
+        self._current_year = current_year
+        
+        # Combined pattern for all date formats
+        # Uses non-capturing groups (?:) and alternation
+        self._date_pattern = re.compile(
+            r'(?:/'  # Path separator
+            r'|[-_])'  # or date separators
+            r'((?:19|20)\d{2})'  # Year group (1900-2099)
+            r'(?:'  # Optional month/day group
+            r'(?:/|[-_])'  # Date separator  
+            r'(?:\d{2})'  # Month
+            r'(?:'  # Optional day
+            r'(?:/|[-_])'  # Date separator
+            r'(?:\d{2})'  # Day
+            r')?'  # Day is optional
+            r')?'  # Month/day group is optional
+        )
+
+    @lru_cache(maxsize=10000)
+    def _extract_year(self, url: str) -> Optional[int]:
+        """Extract the most recent year from URL.
+        
+        Args:
+            url: URL to extract year from
+            
+        Returns:
+            Year as int or None if no valid year found
+        """
+        matches = self._date_pattern.finditer(url)
+        latest_year = None
+        
+        # Find most recent year
+        for match in matches:
+            year = int(match.group(1))
+            if (year <= self._current_year and  # Sanity check
+                (latest_year is None or year > latest_year)):
+                latest_year = year
+                
+        return latest_year
+
+    @lru_cache(maxsize=10000) 
+    def _calculate_score(self, url: str) -> float:
+        """Calculate freshness score based on URL date.
+        
+        More recent years score higher. Uses pre-computed scoring
+        table for common year differences.
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Score between 0.0 and 1.0 * weight
+        """
+        year = self._extract_year(url)
+        if year is None:
+            return 0.5  # Default score
+            
+        # Use lookup table for common year differences
+        year_diff = self._current_year - year
+        if year_diff < len(_FRESHNESS_SCORES):
+            return _FRESHNESS_SCORES[year_diff]
+            
+        # Fallback calculation for older content
+        return max(0.1, 1.0 - year_diff * 0.1)
+
+class DomainAuthorityScorer(URLScorer):
+    __slots__ = ('_weight', '_domain_weights', '_default_weight', '_top_domains')
+    
+    def __init__(
+        self,
+        domain_weights: Dict[str, float],
+        default_weight: float = 0.5,
+        weight: float = 1.0,
+    ):
+        """Initialize domain authority scorer.
+        
+        Args:
+            domain_weights: Dict mapping domains to authority scores
+            default_weight: Score for unknown domains
+            weight: Overall scorer weight multiplier
+            
+        Example:
+            {
+                'python.org': 1.0,
+                'github.com': 0.9,
+                'medium.com': 0.7
+            }
+        """
+        super().__init__(weight=weight)
+        
+        # Pre-process domains for faster lookup
+        self._domain_weights = {
+            domain.lower(): score 
+            for domain, score in domain_weights.items()
+        }
+        self._default_weight = default_weight
+        
+        # Cache top domains for fast path
+        self._top_domains = {
+            domain: score
+            for domain, score in sorted(
+                domain_weights.items(), 
+                key=lambda x: -x[1]
+            )[:5]  # Keep top 5 highest scoring domains
+        }
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _extract_domain(url: str) -> str:
+        """Extract domain from URL ultra-fast.
+        
+        Handles:
+        - Basic domains: "example.com"
+        - Subdomains: "sub.example.com" 
+        - Ports: "example.com:8080"
+        - IPv4: "192.168.1.1"
+        
+        Args:
+            url: Full URL to extract domain from
+            
+        Returns:
+            Lowercase domain without port
+        """
+        # Find domain start
+        start = url.find('://') 
+        if start == -1:
+            start = 0
+        else:
+            start += 3
+            
+        # Find domain end
+        end = url.find('/', start)
+        if end == -1:
+            end = url.find('?', start)
+            if end == -1:
+                end = url.find('#', start)
+                if end == -1:
+                    end = len(url)
+                    
+        # Extract domain and remove port
+        domain = url[start:end]
+        port_idx = domain.rfind(':')
+        if port_idx != -1:
+            domain = domain[:port_idx]
+            
+        return domain.lower()
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate domain authority score.
+        
+        Uses staged approach:
+        1. Check top domains (fastest)
+        2. Check full domain weights
+        3. Return default weight
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Authority score between 0.0 and 1.0 * weight
+        """
+        domain = self._extract_domain(url)
+        
+        # Fast path: check top domains first
+        score = self._top_domains.get(domain)
+        if score is not None:
+            return score
+            
+        # Regular path: check all domains
+        return self._domain_weights.get(domain, self._default_weight)

crawl4ai/docker_client.py

@@ -0,0 +1,170 @@
+from typing import List, Optional, Union, AsyncGenerator, Dict, Any
+import httpx
+import json
+from urllib.parse import urljoin
+import asyncio
+
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from .models import CrawlResult
+from .async_logger import AsyncLogger, LogLevel
+
+
+class Crawl4aiClientError(Exception):
+    """Base exception for Crawl4ai Docker client errors."""
+    pass
+
+
+class ConnectionError(Crawl4aiClientError):
+    """Raised when connection to the Docker server fails."""
+    pass
+
+
+class RequestError(Crawl4aiClientError):
+    """Raised when the server returns an error response."""
+    pass
+
+
+class Crawl4aiDockerClient:
+    """Client for interacting with Crawl4AI Docker server with token authentication."""
+    
+    def __init__(
+        self,
+        base_url: str = "http://localhost:8000",
+        timeout: float = 30.0,
+        verify_ssl: bool = True,
+        verbose: bool = True,
+        log_file: Optional[str] = None
+    ):
+        self.base_url = base_url.rstrip('/')
+        self.timeout = timeout
+        self.logger = AsyncLogger(log_file=log_file, log_level=LogLevel.DEBUG, verbose=verbose)
+        self._http_client = httpx.AsyncClient(
+            timeout=timeout,
+            verify=verify_ssl,
+            headers={"Content-Type": "application/json"}
+        )
+        self._token: Optional[str] = None
+
+    async def authenticate(self, email: str) -> None:
+        """Authenticate with the server and store the token."""
+        url = urljoin(self.base_url, "/token")
+        try:
+            self.logger.info(f"Authenticating with email: {email}", tag="AUTH")
+            response = await self._http_client.post(url, json={"email": email})
+            response.raise_for_status()
+            data = response.json()
+            self._token = data["access_token"]
+            self._http_client.headers["Authorization"] = f"Bearer {self._token}"
+            self.logger.success("Authentication successful", tag="AUTH")
+        except (httpx.RequestError, httpx.HTTPStatusError) as e:
+            error_msg = f"Authentication failed: {str(e)}"
+            self.logger.error(error_msg, tag="ERROR")
+            raise ConnectionError(error_msg)
+
+    async def _check_server(self) -> None:
+        """Check if server is reachable, raising an error if not."""
+        try:
+            await self._http_client.get(urljoin(self.base_url, "/health"))
+            self.logger.success(f"Connected to {self.base_url}", tag="READY")
+        except httpx.RequestError as e:
+            self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR")
+            raise ConnectionError(f"Cannot connect to server: {str(e)}")
+
+    def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, 
+                       crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]:
+        """Prepare request data from configs."""
+        return {
+            "urls": urls,
+            "browser_config": browser_config.dump() if browser_config else {},
+            "crawler_config": crawler_config.dump() if crawler_config else {}
+        }
+
+    async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
+        """Make an HTTP request with error handling."""
+        url = urljoin(self.base_url, endpoint)
+        try:
+            response = await self._http_client.request(method, url, **kwargs)
+            response.raise_for_status()
+            return response
+        except httpx.TimeoutException as e:
+            raise ConnectionError(f"Request timed out: {str(e)}")
+        except httpx.RequestError as e:
+            raise ConnectionError(f"Failed to connect: {str(e)}")
+        except httpx.HTTPStatusError as e:
+            error_msg = (e.response.json().get("detail", str(e)) 
+                        if "application/json" in e.response.headers.get("content-type", "") 
+                        else str(e))
+            raise RequestError(f"Server error {e.response.status_code}: {error_msg}")
+
+    async def crawl(
+        self,
+        urls: List[str],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None
+    ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
+        """Execute a crawl operation."""
+        if not self._token:
+            raise Crawl4aiClientError("Authentication required. Call authenticate() first.")
+        await self._check_server()
+        
+        data = self._prepare_request(urls, browser_config, crawler_config)
+        is_streaming = crawler_config and crawler_config.stream
+        
+        self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL")
+        
+        if is_streaming:
+            async def stream_results() -> AsyncGenerator[CrawlResult, None]:
+                async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response:
+                    response.raise_for_status()
+                    async for line in response.aiter_lines():
+                        if line.strip():
+                            result = json.loads(line)
+                            if "error" in result:
+                                self.logger.error_status(url=result.get("url", "unknown"), error=result["error"])
+                                continue
+                            self.logger.url_status(url=result.get("url", "unknown"), success=True, timing=result.get("timing", 0.0))
+                            if result.get("status") == "completed":
+                                continue
+                            else:
+                                yield CrawlResult(**result)
+            return stream_results()
+        
+        response = await self._request("POST", "/crawl", json=data)
+        result_data = response.json()
+        if not result_data.get("success", False):
+            raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}")
+        
+        results = [CrawlResult(**r) for r in result_data.get("results", [])]
+        self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL")
+        return results[0] if len(results) == 1 else results
+
+    async def get_schema(self) -> Dict[str, Any]:
+        """Retrieve configuration schemas."""
+        if not self._token:
+            raise Crawl4aiClientError("Authentication required. Call authenticate() first.")
+        response = await self._request("GET", "/schema")
+        return response.json()
+
+    async def close(self) -> None:
+        """Close the HTTP client session."""
+        self.logger.info("Closing client", tag="CLOSE")
+        await self._http_client.aclose()
+
+    async def __aenter__(self) -> "Crawl4aiDockerClient":
+        return self
+
+    async def __aexit__(self, exc_type: Optional[type], exc_val: Optional[Exception], exc_tb: Optional[Any]) -> None:
+        await self.close()
+
+
+# Example usage
+async def main():
+    async with Crawl4aiDockerClient(verbose=True) as client:
+        await client.authenticate("user@example.com")
+        result = await client.crawl(["https://example.com"])
+        print(result)
+        schema = await client.get_schema()
+        print(schema)
+
+if __name__ == "__main__":
+    asyncio.run(main())

crawl4ai/html2text/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

crawl4ai/html2text/_typing.py

@@ -0,0 +1,3 @@
+class OutCallback:
+    def __call__(self, s: str) -> None:
+        ...

crawl4ai/html2text/cli.py

@@ -0,0 +1,330 @@
+import argparse
+import sys
+
+from . import HTML2Text, __version__, config
+
+
+def main() -> None:
+    baseurl = ""
+
+    class bcolors:
+        HEADER = "\033[95m"
+        OKBLUE = "\033[94m"
+        OKGREEN = "\033[92m"
+        WARNING = "\033[93m"
+        FAIL = "\033[91m"
+        ENDC = "\033[0m"
+        BOLD = "\033[1m"
+        UNDERLINE = "\033[4m"
+
+    p = argparse.ArgumentParser()
+    p.add_argument(
+        "--default-image-alt",
+        dest="default_image_alt",
+        default=config.DEFAULT_IMAGE_ALT,
+        help="The default alt string for images with missing ones",
+    )
+    p.add_argument(
+        "--pad-tables",
+        dest="pad_tables",
+        action="store_true",
+        default=config.PAD_TABLES,
+        help="pad the cells to equal column width in tables",
+    )
+    p.add_argument(
+        "--no-wrap-links",
+        dest="wrap_links",
+        action="store_false",
+        default=config.WRAP_LINKS,
+        help="don't wrap links during conversion",
+    )
+    p.add_argument(
+        "--wrap-list-items",
+        dest="wrap_list_items",
+        action="store_true",
+        default=config.WRAP_LIST_ITEMS,
+        help="wrap list items during conversion",
+    )
+    p.add_argument(
+        "--wrap-tables",
+        dest="wrap_tables",
+        action="store_true",
+        default=config.WRAP_TABLES,
+        help="wrap tables",
+    )
+    p.add_argument(
+        "--ignore-emphasis",
+        dest="ignore_emphasis",
+        action="store_true",
+        default=config.IGNORE_EMPHASIS,
+        help="don't include any formatting for emphasis",
+    )
+    p.add_argument(
+        "--reference-links",
+        dest="inline_links",
+        action="store_false",
+        default=config.INLINE_LINKS,
+        help="use reference style links instead of inline links",
+    )
+    p.add_argument(
+        "--ignore-links",
+        dest="ignore_links",
+        action="store_true",
+        default=config.IGNORE_ANCHORS,
+        help="don't include any formatting for links",
+    )
+    p.add_argument(
+        "--ignore-mailto-links",
+        action="store_true",
+        dest="ignore_mailto_links",
+        default=config.IGNORE_MAILTO_LINKS,
+        help="don't include mailto: links",
+    )
+    p.add_argument(
+        "--protect-links",
+        dest="protect_links",
+        action="store_true",
+        default=config.PROTECT_LINKS,
+        help="protect links from line breaks surrounding them with angle brackets",
+    )
+    p.add_argument(
+        "--ignore-images",
+        dest="ignore_images",
+        action="store_true",
+        default=config.IGNORE_IMAGES,
+        help="don't include any formatting for images",
+    )
+    p.add_argument(
+        "--images-as-html",
+        dest="images_as_html",
+        action="store_true",
+        default=config.IMAGES_AS_HTML,
+        help=(
+            "Always write image tags as raw html; preserves `height`, `width` and "
+            "`alt` if possible."
+        ),
+    )
+    p.add_argument(
+        "--images-to-alt",
+        dest="images_to_alt",
+        action="store_true",
+        default=config.IMAGES_TO_ALT,
+        help="Discard image data, only keep alt text",
+    )
+    p.add_argument(
+        "--images-with-size",
+        dest="images_with_size",
+        action="store_true",
+        default=config.IMAGES_WITH_SIZE,
+        help=(
+            "Write image tags with height and width attrs as raw html to retain "
+            "dimensions"
+        ),
+    )
+    p.add_argument(
+        "-g",
+        "--google-doc",
+        action="store_true",
+        dest="google_doc",
+        default=False,
+        help="convert an html-exported Google Document",
+    )
+    p.add_argument(
+        "-d",
+        "--dash-unordered-list",
+        action="store_true",
+        dest="ul_style_dash",
+        default=False,
+        help="use a dash rather than a star for unordered list items",
+    )
+    p.add_argument(
+        "-e",
+        "--asterisk-emphasis",
+        action="store_true",
+        dest="em_style_asterisk",
+        default=False,
+        help="use an asterisk rather than an underscore for emphasized text",
+    )
+    p.add_argument(
+        "-b",
+        "--body-width",
+        dest="body_width",
+        type=int,
+        default=config.BODY_WIDTH,
+        help="number of characters per output line, 0 for no wrap",
+    )
+    p.add_argument(
+        "-i",
+        "--google-list-indent",
+        dest="list_indent",
+        type=int,
+        default=config.GOOGLE_LIST_INDENT,
+        help="number of pixels Google indents nested lists",
+    )
+    p.add_argument(
+        "-s",
+        "--hide-strikethrough",
+        action="store_true",
+        dest="hide_strikethrough",
+        default=False,
+        help="hide strike-through text. only relevant when -g is " "specified as well",
+    )
+    p.add_argument(
+        "--escape-all",
+        action="store_true",
+        dest="escape_snob",
+        default=False,
+        help=(
+            "Escape all special characters.  Output is less readable, but avoids "
+            "corner case formatting issues."
+        ),
+    )
+    p.add_argument(
+        "--bypass-tables",
+        action="store_true",
+        dest="bypass_tables",
+        default=config.BYPASS_TABLES,
+        help="Format tables in HTML rather than Markdown syntax.",
+    )
+    p.add_argument(
+        "--ignore-tables",
+        action="store_true",
+        dest="ignore_tables",
+        default=config.IGNORE_TABLES,
+        help="Ignore table-related tags (table, th, td, tr) " "while keeping rows.",
+    )
+    p.add_argument(
+        "--single-line-break",
+        action="store_true",
+        dest="single_line_break",
+        default=config.SINGLE_LINE_BREAK,
+        help=(
+            "Use a single line break after a block element rather than two line "
+            "breaks. NOTE: Requires --body-width=0"
+        ),
+    )
+    p.add_argument(
+        "--unicode-snob",
+        action="store_true",
+        dest="unicode_snob",
+        default=config.UNICODE_SNOB,
+        help="Use unicode throughout document",
+    )
+    p.add_argument(
+        "--no-automatic-links",
+        action="store_false",
+        dest="use_automatic_links",
+        default=config.USE_AUTOMATIC_LINKS,
+        help="Do not use automatic links wherever applicable",
+    )
+    p.add_argument(
+        "--no-skip-internal-links",
+        action="store_false",
+        dest="skip_internal_links",
+        default=config.SKIP_INTERNAL_LINKS,
+        help="Do not skip internal links",
+    )
+    p.add_argument(
+        "--links-after-para",
+        action="store_true",
+        dest="links_each_paragraph",
+        default=config.LINKS_EACH_PARAGRAPH,
+        help="Put links after each paragraph instead of document",
+    )
+    p.add_argument(
+        "--mark-code",
+        action="store_true",
+        dest="mark_code",
+        default=config.MARK_CODE,
+        help="Mark program code blocks with [code]...[/code]",
+    )
+    p.add_argument(
+        "--decode-errors",
+        dest="decode_errors",
+        default=config.DECODE_ERRORS,
+        help=(
+            "What to do in case of decode errors.'ignore', 'strict' and 'replace' are "
+            "acceptable values"
+        ),
+    )
+    p.add_argument(
+        "--open-quote",
+        dest="open_quote",
+        default=config.OPEN_QUOTE,
+        help="The character used to open quotes",
+    )
+    p.add_argument(
+        "--close-quote",
+        dest="close_quote",
+        default=config.CLOSE_QUOTE,
+        help="The character used to close quotes",
+    )
+    p.add_argument(
+        "--version", action="version", version=".".join(map(str, __version__))
+    )
+    p.add_argument("filename", nargs="?")
+    p.add_argument("encoding", nargs="?", default="utf-8")
+    p.add_argument(
+        "--include-sup-sub",
+        dest="include_sup_sub",
+        action="store_true",
+        default=config.INCLUDE_SUP_SUB,
+        help="Include the sup and sub tags",
+    )
+    args = p.parse_args()
+
+    if args.filename and args.filename != "-":
+        with open(args.filename, "rb") as fp:
+            data = fp.read()
+    else:
+        data = sys.stdin.buffer.read()
+
+    try:
+        html = data.decode(args.encoding, args.decode_errors)
+    except UnicodeDecodeError as err:
+        warning = bcolors.WARNING + "Warning:" + bcolors.ENDC
+        warning += " Use the " + bcolors.OKGREEN
+        warning += "--decode-errors=ignore" + bcolors.ENDC + " flag."
+        print(warning)
+        raise err
+
+    h = HTML2Text(baseurl=baseurl)
+    # handle options
+    if args.ul_style_dash:
+        h.ul_item_mark = "-"
+    if args.em_style_asterisk:
+        h.emphasis_mark = "*"
+        h.strong_mark = "__"
+
+    h.body_width = args.body_width
+    h.google_list_indent = args.list_indent
+    h.ignore_emphasis = args.ignore_emphasis
+    h.ignore_links = args.ignore_links
+    h.ignore_mailto_links = args.ignore_mailto_links
+    h.protect_links = args.protect_links
+    h.ignore_images = args.ignore_images
+    h.images_as_html = args.images_as_html
+    h.images_to_alt = args.images_to_alt
+    h.images_with_size = args.images_with_size
+    h.google_doc = args.google_doc
+    h.hide_strikethrough = args.hide_strikethrough
+    h.escape_snob = args.escape_snob
+    h.bypass_tables = args.bypass_tables
+    h.ignore_tables = args.ignore_tables
+    h.single_line_break = args.single_line_break
+    h.inline_links = args.inline_links
+    h.unicode_snob = args.unicode_snob
+    h.use_automatic_links = args.use_automatic_links
+    h.skip_internal_links = args.skip_internal_links
+    h.links_each_paragraph = args.links_each_paragraph
+    h.mark_code = args.mark_code
+    h.wrap_links = args.wrap_links
+    h.wrap_list_items = args.wrap_list_items
+    h.wrap_tables = args.wrap_tables
+    h.pad_tables = args.pad_tables
+    h.default_image_alt = args.default_image_alt
+    h.open_quote = args.open_quote
+    h.close_quote = args.close_quote
+    h.include_sup_sub = args.include_sup_sub
+
+    sys.stdout.write(h.handle(html))

crawl4ai/html2text/config.py

@@ -0,0 +1,172 @@
+import re
+
+# Use Unicode characters instead of their ascii pseudo-replacements
+UNICODE_SNOB = False
+
+# Marker to use for marking tables for padding post processing
+TABLE_MARKER_FOR_PAD = "special_marker_for_table_padding"
+# Escape all special characters.  Output is less readable, but avoids
+# corner case formatting issues.
+ESCAPE_SNOB = False
+ESCAPE_BACKSLASH = False
+ESCAPE_DOT = False
+ESCAPE_PLUS = False
+ESCAPE_DASH = False
+
+# Put the links after each paragraph instead of at the end.
+LINKS_EACH_PARAGRAPH = False
+
+# Wrap long lines at position. 0 for no wrapping.
+BODY_WIDTH = 78
+
+# Don't show internal links (href="#local-anchor") -- corresponding link
+# targets won't be visible in the plain text file anyway.
+SKIP_INTERNAL_LINKS = True
+
+# Use inline, rather than reference, formatting for images and links
+INLINE_LINKS = True
+
+# Protect links from line breaks surrounding them with angle brackets (in
+# addition to their square brackets)
+PROTECT_LINKS = False
+# WRAP_LINKS = True
+WRAP_LINKS = True
+
+# Wrap list items.
+WRAP_LIST_ITEMS = False
+
+# Wrap tables
+WRAP_TABLES = False
+
+# Number of pixels Google indents nested lists
+GOOGLE_LIST_INDENT = 36
+
+# Values Google and others may use to indicate bold text
+BOLD_TEXT_STYLE_VALUES = ("bold", "700", "800", "900")
+
+IGNORE_ANCHORS = False
+IGNORE_MAILTO_LINKS = False
+IGNORE_IMAGES = False
+IMAGES_AS_HTML = False
+IMAGES_TO_ALT = False
+IMAGES_WITH_SIZE = False
+IGNORE_EMPHASIS = False
+MARK_CODE = False
+DECODE_ERRORS = "strict"
+DEFAULT_IMAGE_ALT = ""
+PAD_TABLES = False
+
+# Convert links with same href and text to <href> format
+# if they are absolute links
+USE_AUTOMATIC_LINKS = True
+
+# For checking space-only lines on line 771
+RE_SPACE = re.compile(r"\s\+")
+
+RE_ORDERED_LIST_MATCHER = re.compile(r"\d+\.\s")
+RE_UNORDERED_LIST_MATCHER = re.compile(r"[-\*\+]\s")
+RE_MD_CHARS_MATCHER = re.compile(r"([\\\[\]\(\)])")
+RE_MD_CHARS_MATCHER_ALL = re.compile(r"([`\*_{}\[\]\(\)#!])")
+
+# to find links in the text
+RE_LINK = re.compile(r"(\[.*?\] ?\(.*?\))|(\[.*?\]:.*?)")
+
+# to find table separators
+RE_TABLE = re.compile(r" \| ")
+
+RE_MD_DOT_MATCHER = re.compile(
+    r"""
+    ^             # start of line
+    (\s*\d+)      # optional whitespace and a number
+    (\.)          # dot
+    (?=\s)        # lookahead assert whitespace
+    """,
+    re.MULTILINE | re.VERBOSE,
+)
+RE_MD_PLUS_MATCHER = re.compile(
+    r"""
+    ^
+    (\s*)
+    (\+)
+    (?=\s)
+    """,
+    flags=re.MULTILINE | re.VERBOSE,
+)
+RE_MD_DASH_MATCHER = re.compile(
+    r"""
+    ^
+    (\s*)
+    (-)
+    (?=\s|\-)     # followed by whitespace (bullet list, or spaced out hr)
+                  # or another dash (header or hr)
+    """,
+    flags=re.MULTILINE | re.VERBOSE,
+)
+RE_SLASH_CHARS = r"\`*_{}[]()#+-.!"
+RE_MD_BACKSLASH_MATCHER = re.compile(
+    r"""
+    (\\)          # match one slash
+    (?=[%s])      # followed by a char that requires escaping
+    """
+    % re.escape(RE_SLASH_CHARS),
+    flags=re.VERBOSE,
+)
+
+UNIFIABLE = {
+    "rsquo": "'",
+    "lsquo": "'",
+    "rdquo": '"',
+    "ldquo": '"',
+    "copy": "(C)",
+    "mdash": "--",
+    "nbsp": " ",
+    "rarr": "->",
+    "larr": "<-",
+    "middot": "*",
+    "ndash": "-",
+    "oelig": "oe",
+    "aelig": "ae",
+    "agrave": "a",
+    "aacute": "a",
+    "acirc": "a",
+    "atilde": "a",
+    "auml": "a",
+    "aring": "a",
+    "egrave": "e",
+    "eacute": "e",
+    "ecirc": "e",
+    "euml": "e",
+    "igrave": "i",
+    "iacute": "i",
+    "icirc": "i",
+    "iuml": "i",
+    "ograve": "o",
+    "oacute": "o",
+    "ocirc": "o",
+    "otilde": "o",
+    "ouml": "o",
+    "ugrave": "u",
+    "uacute": "u",
+    "ucirc": "u",
+    "uuml": "u",
+    "lrm": "",
+    "rlm": "",
+}
+
+# Format tables in HTML rather than Markdown syntax
+BYPASS_TABLES = False
+# Ignore table-related tags (table, th, td, tr) while keeping rows
+IGNORE_TABLES = False
+
+
+# Use a single line break after a block element rather than two line breaks.
+# NOTE: Requires body width setting to be 0.
+SINGLE_LINE_BREAK = False
+
+
+# Use double quotation marks when converting the <q> tag.
+OPEN_QUOTE = '"'
+CLOSE_QUOTE = '"'
+
+# Include the <sup> and <sub> tags
+INCLUDE_SUP_SUB = False

crawl4ai/html2text/elements.py

@@ -0,0 +1,18 @@
+from typing import Dict, Optional
+
+
+class AnchorElement:
+    __slots__ = ["attrs", "count", "outcount"]
+
+    def __init__(self, attrs: Dict[str, Optional[str]], count: int, outcount: int):
+        self.attrs = attrs
+        self.count = count
+        self.outcount = outcount
+
+
+class ListElement:
+    __slots__ = ["name", "num"]
+
+    def __init__(self, name: str, num: int):
+        self.name = name
+        self.num = num

crawl4ai/html2text/utils.py

@@ -0,0 +1,304 @@
+import html.entities
+from typing import Dict, List, Optional
+
+from . import config
+
+unifiable_n = {
+    html.entities.name2codepoint[k]: v
+    for k, v in config.UNIFIABLE.items()
+    if k != "nbsp"
+}
+
+
+def hn(tag: str) -> int:
+    if tag[0] == "h" and len(tag) == 2:
+        n = tag[1]
+        if "0" < n <= "9":
+            return int(n)
+    return 0
+
+
+def dumb_property_dict(style: str) -> Dict[str, str]:
+    """
+    :returns: A hash of css attributes
+    """
+    return {
+        x.strip().lower(): y.strip().lower()
+        for x, y in [z.split(":", 1) for z in style.split(";") if ":" in z]
+    }
+
+
+def dumb_css_parser(data: str) -> Dict[str, Dict[str, str]]:
+    """
+    :type data: str
+
+    :returns: A hash of css selectors, each of which contains a hash of
+    css attributes.
+    :rtype: dict
+    """
+    # remove @import sentences
+    data += ";"
+    importIndex = data.find("@import")
+    while importIndex != -1:
+        data = data[0:importIndex] + data[data.find(";", importIndex) + 1 :]
+        importIndex = data.find("@import")
+
+    # parse the css. reverted from dictionary comprehension in order to
+    # support older pythons
+    pairs = [x.split("{") for x in data.split("}") if "{" in x.strip()]
+    try:
+        elements = {a.strip(): dumb_property_dict(b) for a, b in pairs}
+    except ValueError:
+        elements = {}  # not that important
+
+    return elements
+
+
+def element_style(
+    attrs: Dict[str, Optional[str]],
+    style_def: Dict[str, Dict[str, str]],
+    parent_style: Dict[str, str],
+) -> Dict[str, str]:
+    """
+    :type attrs: dict
+    :type style_def: dict
+    :type style_def: dict
+
+    :returns: A hash of the 'final' style attributes of the element
+    :rtype: dict
+    """
+    style = parent_style.copy()
+    if "class" in attrs:
+        assert attrs["class"] is not None
+        for css_class in attrs["class"].split():
+            css_style = style_def.get("." + css_class, {})
+            style.update(css_style)
+    if "style" in attrs:
+        assert attrs["style"] is not None
+        immediate_style = dumb_property_dict(attrs["style"])
+        style.update(immediate_style)
+
+    return style
+
+
+def google_list_style(style: Dict[str, str]) -> str:
+    """
+    Finds out whether this is an ordered or unordered list
+
+    :type style: dict
+
+    :rtype: str
+    """
+    if "list-style-type" in style:
+        list_style = style["list-style-type"]
+        if list_style in ["disc", "circle", "square", "none"]:
+            return "ul"
+
+    return "ol"
+
+
+def google_has_height(style: Dict[str, str]) -> bool:
+    """
+    Check if the style of the element has the 'height' attribute
+    explicitly defined
+
+    :type style: dict
+
+    :rtype: bool
+    """
+    return "height" in style
+
+
+def google_text_emphasis(style: Dict[str, str]) -> List[str]:
+    """
+    :type style: dict
+
+    :returns: A list of all emphasis modifiers of the element
+    :rtype: list
+    """
+    emphasis = []
+    if "text-decoration" in style:
+        emphasis.append(style["text-decoration"])
+    if "font-style" in style:
+        emphasis.append(style["font-style"])
+    if "font-weight" in style:
+        emphasis.append(style["font-weight"])
+
+    return emphasis
+
+
+def google_fixed_width_font(style: Dict[str, str]) -> bool:
+    """
+    Check if the css of the current element defines a fixed width font
+
+    :type style: dict
+
+    :rtype: bool
+    """
+    font_family = ""
+    if "font-family" in style:
+        font_family = style["font-family"]
+    return "courier new" == font_family or "consolas" == font_family
+
+
+def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
+    """
+    Extract numbering from list element attributes
+
+    :type attrs: dict
+
+    :rtype: int or None
+    """
+    if "start" in attrs:
+        assert attrs["start"] is not None
+        try:
+            return int(attrs["start"]) - 1
+        except ValueError:
+            pass
+
+    return 0
+
+
+def skipwrap(
+    para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool
+) -> bool:
+    # If it appears to contain a link
+    # don't wrap
+    if not wrap_links and config.RE_LINK.search(para):
+        return True
+    # If the text begins with four spaces or one tab, it's a code block;
+    # don't wrap
+    if para[0:4] == "    " or para[0] == "\t":
+        return True
+
+    # If the text begins with only two "--", possibly preceded by
+    # whitespace, that's an emdash; so wrap.
+    stripped = para.lstrip()
+    if stripped[0:2] == "--" and len(stripped) > 2 and stripped[2] != "-":
+        return False
+
+    # I'm not sure what this is for; I thought it was to detect lists,
+    # but there's a <br>-inside-<span> case in one of the tests that
+    # also depends upon it.
+    if stripped[0:1] in ("-", "*") and not stripped[0:2] == "**":
+        return not wrap_list_items
+
+    # If text contains a pipe character it is likely a table
+    if not wrap_tables and config.RE_TABLE.search(para):
+        return True
+
+    # If the text begins with a single -, *, or +, followed by a space,
+    # or an integer, followed by a ., followed by a space (in either
+    # case optionally proceeded by whitespace), it's a list; don't wrap.
+    return bool(
+        config.RE_ORDERED_LIST_MATCHER.match(stripped)
+        or config.RE_UNORDERED_LIST_MATCHER.match(stripped)
+    )
+
+
+def escape_md(text: str) -> str:
+    """
+    Escapes markdown-sensitive characters within other markdown
+    constructs.
+    """
+    return config.RE_MD_CHARS_MATCHER.sub(r"\\\1", text)
+
+
+def escape_md_section(
+    text: str,
+    escape_backslash: bool = True,
+    snob: bool = False,
+    escape_dot: bool = True,
+    escape_plus: bool = True,
+    escape_dash: bool = True,
+) -> str:
+    """
+    Escapes markdown-sensitive characters across whole document sections.
+    Each escaping operation can be controlled individually.
+    """
+    if escape_backslash:
+        text = config.RE_MD_BACKSLASH_MATCHER.sub(r"\\\1", text)
+
+    if snob:
+        text = config.RE_MD_CHARS_MATCHER_ALL.sub(r"\\\1", text)
+
+    if escape_dot:
+        text = config.RE_MD_DOT_MATCHER.sub(r"\1\\\2", text)
+
+    if escape_plus:
+        text = config.RE_MD_PLUS_MATCHER.sub(r"\1\\\2", text)
+
+    if escape_dash:
+        text = config.RE_MD_DASH_MATCHER.sub(r"\1\\\2", text)
+
+    return text
+
+
+def reformat_table(lines: List[str], right_margin: int) -> List[str]:
+    """
+    Given the lines of a table
+    padds the cells and returns the new lines
+    """
+    # find the maximum width of the columns
+    max_width = [len(x.rstrip()) + right_margin for x in lines[0].split("|")]
+    max_cols = len(max_width)
+    for line in lines:
+        cols = [x.rstrip() for x in line.split("|")]
+        num_cols = len(cols)
+
+        # don't drop any data if colspan attributes result in unequal lengths
+        if num_cols < max_cols:
+            cols += [""] * (max_cols - num_cols)
+        elif max_cols < num_cols:
+            max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
+            max_cols = num_cols
+
+        max_width = [
+            max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
+        ]
+
+    # reformat
+    new_lines = []
+    for line in lines:
+        cols = [x.rstrip() for x in line.split("|")]
+        if set(line.strip()) == set("-|"):
+            filler = "-"
+            new_cols = [
+                x.rstrip() + (filler * (M - len(x.rstrip())))
+                for x, M in zip(cols, max_width)
+            ]
+            new_lines.append("|-" + "|".join(new_cols) + "|")
+        else:
+            filler = " "
+            new_cols = [
+                x.rstrip() + (filler * (M - len(x.rstrip())))
+                for x, M in zip(cols, max_width)
+            ]
+            new_lines.append("| " + "|".join(new_cols) + "|")
+    return new_lines
+
+
+def pad_tables_in_text(text: str, right_margin: int = 1) -> str:
+    """
+    Provide padding for tables in the text
+    """
+    lines = text.split("\n")
+    table_buffer = []  # type: List[str]
+    table_started = False
+    new_lines = []
+    for line in lines:
+        # Toggle table started
+        if config.TABLE_MARKER_FOR_PAD in line:
+            table_started = not table_started
+            if not table_started:
+                table = reformat_table(table_buffer, right_margin)
+                new_lines.extend(table)
+                table_buffer = []
+                new_lines.append("")
+            continue
+        # Process lines
+        if table_started:
+            table_buffer.append(line)
+        else:
+            new_lines.append(line)
+    return "\n".join(new_lines)

crawl4ai/hub.py

@@ -0,0 +1,69 @@
+# crawl4ai/hub.py
+from abc import ABC, abstractmethod
+from typing import Dict, Type, Union
+import logging
+import importlib
+from pathlib import Path
+import inspect
+
+logger = logging.getLogger(__name__)
+
+
+class BaseCrawler(ABC):
+    def __init__(self):
+        self.logger = logging.getLogger(self.__class__.__name__)
+        
+    @abstractmethod
+    async def run(self, url: str = "", **kwargs) -> str:
+        """
+        Implement this method to return JSON string.
+        Must accept URL + arbitrary kwargs for flexibility.
+        """
+        pass
+
+    def __init_subclass__(cls, **kwargs):
+        """Enforce interface validation on subclassing"""
+        super().__init_subclass__(**kwargs)
+        
+        # Verify run method signature
+        run_method = cls.run
+        if not run_method.__code__.co_argcount >= 2:  # self + url
+            raise TypeError(f"{cls.__name__} must implement 'run(self, url: str, **kwargs)'")
+            
+        # Verify async nature
+        if not inspect.iscoroutinefunction(run_method):
+            raise TypeError(f"{cls.__name__}.run must be async")
+
+class CrawlerHub:
+    _crawlers: Dict[str, Type[BaseCrawler]] = {}
+
+    @classmethod
+    def _discover_crawlers(cls):
+        """Dynamically load crawlers from /crawlers in 3 lines"""
+        base_path = Path(__file__).parent / "crawlers"
+        for crawler_dir in base_path.iterdir():
+            if crawler_dir.is_dir():
+                try:
+                    module = importlib.import_module(
+                        f"crawl4ai.crawlers.{crawler_dir.name}.crawler"
+                    )
+                    for attr in dir(module):
+                        cls._maybe_register_crawler(
+                            getattr(module, attr), crawler_dir.name
+                        )
+                except Exception as e:
+                    logger.warning(f"Failed {crawler_dir.name}: {str(e)}")
+
+    @classmethod
+    def _maybe_register_crawler(cls, obj, name: str):
+        """Brilliant one-liner registration"""
+        if isinstance(obj, type) and issubclass(obj, BaseCrawler) and obj != BaseCrawler:
+            module = importlib.import_module(obj.__module__)
+            obj.meta = getattr(module, "__meta__", {})
+            cls._crawlers[name] = obj
+
+    @classmethod
+    def get(cls, name: str) -> Union[Type[BaseCrawler], None]:
+        if not cls._crawlers:
+            cls._discover_crawlers()
+        return cls._crawlers.get(name)

crawl4ai/install.py

@@ -0,0 +1,171 @@
+import subprocess
+import sys
+import asyncio
+from .async_logger import AsyncLogger, LogLevel
+from pathlib import Path
+import os
+import shutil
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+def setup_home_directory():
+    """Set up the .crawl4ai folder structure in the user's home directory."""
+    base_dir = os.getenv("CRAWL4_AI_BASE_DIRECTORY")
+    crawl4ai_folder = Path(base_dir) if base_dir else Path.home()
+    crawl4ai_config = crawl4ai_folder / "global.yml"
+    crawl4ai_folder = crawl4ai_folder / ".crawl4ai"
+    cache_folder = crawl4ai_folder / "cache"
+    content_folders = [
+        "html_content",
+        "cleaned_html",
+        "markdown_content",
+        "extracted_content",
+        "screenshots",
+    ]
+
+    # Clean up old cache if exists
+    if cache_folder.exists():
+        shutil.rmtree(cache_folder)
+
+    # Create new folder structure
+    crawl4ai_folder.mkdir(exist_ok=True)
+    cache_folder.mkdir(exist_ok=True)
+    for folder in content_folders:
+        (crawl4ai_folder / folder).mkdir(exist_ok=True)
+    
+    # If config file does not exist, create it
+    if not crawl4ai_config.exists():
+        with open(crawl4ai_config, "w") as f:
+            f.write("")
+
+def post_install():
+    """Run all post-installation tasks"""
+    logger.info("Running post-installation setup...", tag="INIT")
+    setup_home_directory()
+    install_playwright()
+    run_migration()
+    # TODO: Will be added in the future
+    # setup_builtin_browser()
+    logger.success("Post-installation setup completed!", tag="COMPLETE")
+    
+def setup_builtin_browser():
+    """Set up a builtin browser for use with Crawl4AI"""
+    try:
+        logger.info("Setting up builtin browser...", tag="INIT")
+        asyncio.run(_setup_builtin_browser())
+        logger.success("Builtin browser setup completed!", tag="COMPLETE")
+    except Exception as e:
+        logger.warning(f"Failed to set up builtin browser: {e}")
+        logger.warning("You can manually set up a builtin browser using 'crawl4ai-doctor builtin-browser-start'")
+    
+async def _setup_builtin_browser():
+    try:
+        # Import BrowserProfiler here to avoid circular imports
+        from .browser_profiler import BrowserProfiler
+        profiler = BrowserProfiler(logger=logger)
+        
+        # Launch the builtin browser
+        cdp_url = await profiler.launch_builtin_browser(headless=True)
+        if cdp_url:
+            logger.success(f"Builtin browser launched at {cdp_url}", tag="BROWSER")
+        else:
+            logger.warning("Failed to launch builtin browser", tag="BROWSER")
+    except Exception as e:
+        logger.warning(f"Error setting up builtin browser: {e}", tag="BROWSER")
+        raise
+
+
+def install_playwright():
+    logger.info("Installing Playwright browsers...", tag="INIT")
+    try:
+        # subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chrome"])
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                "playwright",
+                "install",
+                "--with-deps",
+                "--force",
+                "chromium",
+            ]
+        )
+        logger.success(
+            "Playwright installation completed successfully.", tag="COMPLETE"
+        )
+    except subprocess.CalledProcessError:
+        # logger.error(f"Error during Playwright installation: {e}", tag="ERROR")
+        logger.warning(
+            f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
+        )
+    except Exception:
+        # logger.error(f"Unexpected error during Playwright installation: {e}", tag="ERROR")
+        logger.warning(
+            f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
+        )
+
+
+def run_migration():
+    """Initialize database during installation"""
+    try:
+        logger.info("Starting database initialization...", tag="INIT")
+        from crawl4ai.async_database import async_db_manager
+
+        asyncio.run(async_db_manager.initialize())
+        logger.success(
+            "Database initialization completed successfully.", tag="COMPLETE"
+        )
+    except ImportError:
+        logger.warning("Database module not found. Will initialize on first use.")
+    except Exception as e:
+        logger.warning(f"Database initialization failed: {e}")
+        logger.warning("Database will be initialized on first use")
+
+
+async def run_doctor():
+    """Test if Crawl4AI is working properly"""
+    logger.info("Running Crawl4AI health check...", tag="INIT")
+    try:
+        from .async_webcrawler import (
+            AsyncWebCrawler,
+            BrowserConfig,
+            CrawlerRunConfig,
+            CacheMode,
+        )
+
+        browser_config = BrowserConfig(
+            headless=True,
+            browser_type="chromium",
+            ignore_https_errors=True,
+            light_mode=True,
+            viewport_width=1280,
+            viewport_height=720,
+        )
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            screenshot=True,
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            logger.info("Testing crawling capabilities...", tag="TEST")
+            result = await crawler.arun(url="https://crawl4ai.com", config=run_config)
+
+            if result and result.markdown:
+                logger.success("✅ Crawling test passed!", tag="COMPLETE")
+                return True
+            else:
+                raise Exception("Failed to get content")
+
+    except Exception as e:
+        logger.error(f"❌ Test failed: {e}", tag="ERROR")
+        return False
+
+
+def doctor():
+    """Entry point for the doctor command"""
+    import asyncio
+
+    asyncio.run(run_doctor())
+    sys.exit(0)

crawl4ai/js_snippet/__init__.py

@@ -0,0 +1,18 @@
+import os
+
+
+# Create a function get name of a js script, then load from the CURRENT folder of this script and return its content as string, make sure its error free
+def load_js_script(script_name):
+    # Get the path of the current script
+    current_script_path = os.path.dirname(os.path.realpath(__file__))
+    # Get the path of the script to load
+    script_path = os.path.join(current_script_path, script_name + ".js")
+    # Check if the script exists
+    if not os.path.exists(script_path):
+        raise ValueError(
+            f"Script {script_name} not found in the folder {current_script_path}"
+        )
+    # Load the content of the script
+    with open(script_path, "r") as f:
+        script_content = f.read()
+    return script_content

crawl4ai/js_snippet/navigator_overrider.js

@@ -0,0 +1,25 @@
+// Pass the Permissions Test.
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) =>
+    parameters.name === "notifications"
+        ? Promise.resolve({ state: Notification.permission })
+        : originalQuery(parameters);
+Object.defineProperty(navigator, "webdriver", {
+    get: () => undefined,
+});
+window.navigator.chrome = {
+    runtime: {},
+    // Add other properties if necessary
+};
+Object.defineProperty(navigator, "plugins", {
+    get: () => [1, 2, 3, 4, 5],
+});
+Object.defineProperty(navigator, "languages", {
+    get: () => ["en-US", "en"],
+});
+Object.defineProperty(document, "hidden", {
+    get: () => false,
+});
+Object.defineProperty(document, "visibilityState", {
+    get: () => "visible",
+});

crawl4ai/js_snippet/remove_overlay_elements.js

@@ -0,0 +1,119 @@
+async () => {
+    // Function to check if element is visible
+    const isVisible = (elem) => {
+        const style = window.getComputedStyle(elem);
+        return style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0";
+    };
+
+    // Common selectors for popups and overlays
+    const commonSelectors = [
+        // Close buttons first
+        'button[class*="close" i]',
+        'button[class*="dismiss" i]',
+        'button[aria-label*="close" i]',
+        'button[title*="close" i]',
+        'a[class*="close" i]',
+        'span[class*="close" i]',
+
+        // Cookie notices
+        '[class*="cookie-banner" i]',
+        '[id*="cookie-banner" i]',
+        '[class*="cookie-consent" i]',
+        '[id*="cookie-consent" i]',
+
+        // Newsletter/subscription dialogs
+        '[class*="newsletter" i]',
+        '[class*="subscribe" i]',
+
+        // Generic popups/modals
+        '[class*="popup" i]',
+        '[class*="modal" i]',
+        '[class*="overlay" i]',
+        '[class*="dialog" i]',
+        '[role="dialog"]',
+        '[role="alertdialog"]',
+    ];
+
+    // Try to click close buttons first
+    for (const selector of commonSelectors.slice(0, 6)) {
+        const closeButtons = document.querySelectorAll(selector);
+        for (const button of closeButtons) {
+            if (isVisible(button)) {
+                try {
+                    button.click();
+                    await new Promise((resolve) => setTimeout(resolve, 100));
+                } catch (e) {
+                    console.log("Error clicking button:", e);
+                }
+            }
+        }
+    }
+
+    // Remove remaining overlay elements
+    const removeOverlays = () => {
+        // Find elements with high z-index
+        const allElements = document.querySelectorAll("*");
+        for (const elem of allElements) {
+            const style = window.getComputedStyle(elem);
+            const zIndex = parseInt(style.zIndex);
+            const position = style.position;
+
+            if (
+                isVisible(elem) &&
+                (zIndex > 999 || position === "fixed" || position === "absolute") &&
+                (elem.offsetWidth > window.innerWidth * 0.5 ||
+                    elem.offsetHeight > window.innerHeight * 0.5 ||
+                    style.backgroundColor.includes("rgba") ||
+                    parseFloat(style.opacity) < 1)
+            ) {
+                elem.remove();
+            }
+        }
+
+        // Remove elements matching common selectors
+        for (const selector of commonSelectors) {
+            const elements = document.querySelectorAll(selector);
+            elements.forEach((elem) => {
+                if (isVisible(elem)) {
+                    elem.remove();
+                }
+            });
+        }
+    };
+
+    // Remove overlay elements
+    removeOverlays();
+
+    // Remove any fixed/sticky position elements at the top/bottom
+    const removeFixedElements = () => {
+        const elements = document.querySelectorAll("*");
+        elements.forEach((elem) => {
+            const style = window.getComputedStyle(elem);
+            if ((style.position === "fixed" || style.position === "sticky") && isVisible(elem)) {
+                elem.remove();
+            }
+        });
+    };
+
+    removeFixedElements();
+
+    // Remove empty block elements as: div, p, span, etc.
+    const removeEmptyBlockElements = () => {
+        const blockElements = document.querySelectorAll(
+            "div, p, span, section, article, header, footer, aside, nav, main, ul, ol, li, dl, dt, dd, h1, h2, h3, h4, h5, h6"
+        );
+        blockElements.forEach((elem) => {
+            if (elem.innerText.trim() === "") {
+                elem.remove();
+            }
+        });
+    };
+
+    // Remove margin-right and padding-right from body (often added by modal scripts)
+    document.body.style.marginRight = "0px";
+    document.body.style.paddingRight = "0px";
+    document.body.style.overflow = "auto";
+
+    // Wait a bit for any animations to complete
+    await new Promise((resolve) => setTimeout(resolve, 100));
+};

crawl4ai/js_snippet/update_image_dimensions.js

@@ -0,0 +1,54 @@
+() => {
+    return new Promise((resolve) => {
+        const filterImage = (img) => {
+            // Filter out images that are too small
+            if (img.width < 100 && img.height < 100) return false;
+
+            // Filter out images that are not visible
+            const rect = img.getBoundingClientRect();
+            if (rect.width === 0 || rect.height === 0) return false;
+
+            // Filter out images with certain class names (e.g., icons, thumbnails)
+            if (img.classList.contains("icon") || img.classList.contains("thumbnail")) return false;
+
+            // Filter out images with certain patterns in their src (e.g., placeholder images)
+            if (img.src.includes("placeholder") || img.src.includes("icon")) return false;
+
+            return true;
+        };
+
+        const images = Array.from(document.querySelectorAll("img")).filter(filterImage);
+        let imagesLeft = images.length;
+
+        if (imagesLeft === 0) {
+            resolve();
+            return;
+        }
+
+        const checkImage = (img) => {
+            if (img.complete && img.naturalWidth !== 0) {
+                img.setAttribute("width", img.naturalWidth);
+                img.setAttribute("height", img.naturalHeight);
+                imagesLeft--;
+                if (imagesLeft === 0) resolve();
+            }
+        };
+
+        images.forEach((img) => {
+            checkImage(img);
+            if (!img.complete) {
+                img.onload = () => {
+                    checkImage(img);
+                };
+                img.onerror = () => {
+                    imagesLeft--;
+                    if (imagesLeft === 0) resolve();
+                };
+            }
+        });
+
+        // Fallback timeout of 5 seconds
+        // setTimeout(() => resolve(), 5000);
+        resolve();
+    });
+};

crawl4ai/legacy/cli.py

@@ -0,0 +1,123 @@
+import click
+import sys
+import asyncio
+from typing import List
+from .docs_manager import DocsManager
+from .async_logger import AsyncLogger
+
+logger = AsyncLogger(verbose=True)
+docs_manager = DocsManager(logger)
+
+
+def print_table(headers: List[str], rows: List[List[str]], padding: int = 2):
+    """Print formatted table with headers and rows"""
+    widths = [max(len(str(cell)) for cell in col) for col in zip(headers, *rows)]
+    border = "+" + "+".join("-" * (w + 2 * padding) for w in widths) + "+"
+
+    def format_row(row):
+        return (
+            "|"
+            + "|".join(
+                f"{' ' * padding}{str(cell):<{w}}{' ' * padding}"
+                for cell, w in zip(row, widths)
+            )
+            + "|"
+        )
+
+    click.echo(border)
+    click.echo(format_row(headers))
+    click.echo(border)
+    for row in rows:
+        click.echo(format_row(row))
+    click.echo(border)
+
+
+@click.group()
+def cli():
+    """Crawl4AI Command Line Interface"""
+    pass
+
+
+@cli.group()
+def docs():
+    """Documentation operations"""
+    pass
+
+
+@docs.command()
+@click.argument("sections", nargs=-1)
+@click.option(
+    "--mode", type=click.Choice(["extended", "condensed"]), default="extended"
+)
+def combine(sections: tuple, mode: str):
+    """Combine documentation sections"""
+    try:
+        asyncio.run(docs_manager.ensure_docs_exist())
+        click.echo(docs_manager.generate(sections, mode))
+    except Exception as e:
+        logger.error(str(e), tag="ERROR")
+        sys.exit(1)
+
+
+@docs.command()
+@click.argument("query")
+@click.option("--top-k", "-k", default=5)
+@click.option("--build-index", is_flag=True, help="Build index if missing")
+def search(query: str, top_k: int, build_index: bool):
+    """Search documentation"""
+    try:
+        result = docs_manager.search(query, top_k)
+        if result == "No search index available. Call build_search_index() first.":
+            if build_index or click.confirm("No search index found. Build it now?"):
+                asyncio.run(docs_manager.llm_text.generate_index_files())
+                result = docs_manager.search(query, top_k)
+        click.echo(result)
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+@docs.command()
+def update():
+    """Update docs from GitHub"""
+    try:
+        asyncio.run(docs_manager.fetch_docs())
+        click.echo("Documentation updated successfully")
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+@docs.command()
+@click.option("--force-facts", is_flag=True, help="Force regenerate fact files")
+@click.option("--clear-cache", is_flag=True, help="Clear BM25 cache")
+def index(force_facts: bool, clear_cache: bool):
+    """Build or rebuild search indexes"""
+    try:
+        asyncio.run(docs_manager.ensure_docs_exist())
+        asyncio.run(
+            docs_manager.llm_text.generate_index_files(
+                force_generate_facts=force_facts, clear_bm25_cache=clear_cache
+            )
+        )
+        click.echo("Search indexes built successfully")
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+# Add docs list command
+@docs.command()
+def list():
+    """List available documentation sections"""
+    try:
+        sections = docs_manager.list()
+        print_table(["Sections"], [[section] for section in sections])
+
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    cli()

crawl4ai/legacy/crawler_strategy.py

@@ -6,61 +6,62 @@ from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.options import Options
 from selenium.common.exceptions import InvalidArgumentException, WebDriverException
-from selenium.webdriver.chrome.service import Service as ChromeService
-from webdriver_manager.chrome import ChromeDriverManager
+# from selenium.webdriver.chrome.service import Service as ChromeService
+# from webdriver_manager.chrome import ChromeDriverManager
+# from urllib3.exceptions import MaxRetryError
 
-import logging
+from .config import *
+import logging, time
 import base64
 from PIL import Image, ImageDraw, ImageFont
 from io import BytesIO
-from typing import List, Callable
+from typing import Callable
 import requests
 import os
 from pathlib import Path
-from .utils import wrap_text
+from .utils import *
 
-logger = logging.getLogger('selenium.webdriver.remote.remote_connection')
+logger = logging.getLogger("selenium.webdriver.remote.remote_connection")
 logger.setLevel(logging.WARNING)
 
-logger_driver = logging.getLogger('selenium.webdriver.common.service')
+logger_driver = logging.getLogger("selenium.webdriver.common.service")
 logger_driver.setLevel(logging.WARNING)
 
-urllib3_logger = logging.getLogger('urllib3.connectionpool')
+urllib3_logger = logging.getLogger("urllib3.connectionpool")
 urllib3_logger.setLevel(logging.WARNING)
 
 # Disable http.client logging
-http_client_logger = logging.getLogger('http.client')
+http_client_logger = logging.getLogger("http.client")
 http_client_logger.setLevel(logging.WARNING)
 
 # Disable driver_finder and service logging
-driver_finder_logger = logging.getLogger('selenium.webdriver.common.driver_finder')
+driver_finder_logger = logging.getLogger("selenium.webdriver.common.driver_finder")
 driver_finder_logger.setLevel(logging.WARNING)
 
 
-
-
 class CrawlerStrategy(ABC):
     @abstractmethod
     def crawl(self, url: str, **kwargs) -> str:
         pass
-    
+
     @abstractmethod
     def take_screenshot(self, save_path: str):
         pass
-    
+
     @abstractmethod
     def update_user_agent(self, user_agent: str):
         pass
-    
+
     @abstractmethod
     def set_hook(self, hook_type: str, hook: Callable):
         pass
 
+
 class CloudCrawlerStrategy(CrawlerStrategy):
-    def __init__(self, use_cached_html = False):
+    def __init__(self, use_cached_html=False):
         super().__init__()
         self.use_cached_html = use_cached_html
-        
+
     def crawl(self, url: str) -> str:
         data = {
             "urls": [url],
@@ -72,7 +73,8 @@ class CloudCrawlerStrategy(CrawlerStrategy):
         response = requests.post("http://crawl4ai.uccode.io/crawl", json=data)
         response = response.json()
         html = response["results"][0]["html"]
-        return html
+        return sanitize_input_encode(html)
+
 
 class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
     def __init__(self, use_cached_html=False, js_code=None, **kwargs):
@@ -80,23 +82,30 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         print("[LOG] 🚀 Initializing LocalSeleniumCrawlerStrategy")
         self.options = Options()
         self.options.headless = True
+        if kwargs.get("proxy"):
+            self.options.add_argument("--proxy-server={}".format(kwargs.get("proxy")))
         if kwargs.get("user_agent"):
             self.options.add_argument("--user-agent=" + kwargs.get("user_agent"))
         else:
-            user_agent = kwargs.get("user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+            user_agent = kwargs.get(
+                "user_agent",
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            )
             self.options.add_argument(f"--user-agent={user_agent}")
-            self.options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
-                  
+            self.options.add_argument(
+                "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+            )
+
         self.options.headless = kwargs.get("headless", True)
         if self.options.headless:
             self.options.add_argument("--headless")
-        
-        self.options.add_argument("--disable-gpu")  
+
+        self.options.add_argument("--disable-gpu")
         self.options.add_argument("--window-size=1920,1080")
         self.options.add_argument("--no-sandbox")
         self.options.add_argument("--disable-dev-shm-usage")
-        self.options.add_argument("--disable-blink-features=AutomationControlled")     
-        
+        self.options.add_argument("--disable-blink-features=AutomationControlled")
+
         # self.options.add_argument("--disable-dev-shm-usage")
         self.options.add_argument("--disable-gpu")
         # self.options.add_argument("--disable-extensions")
@@ -116,43 +125,45 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         self.use_cached_html = use_cached_html
         self.js_code = js_code
         self.verbose = kwargs.get("verbose", False)
-        
+
         # Hooks
         self.hooks = {
-            'on_driver_created': None,
-            'on_user_agent_updated': None,
-            'before_get_url': None,
-            'after_get_url': None,
-            'before_return_html': None
+            "on_driver_created": None,
+            "on_user_agent_updated": None,
+            "before_get_url": None,
+            "after_get_url": None,
+            "before_return_html": None,
         }
 
         # chromedriver_autoinstaller.install()
         # import chromedriver_autoinstaller
-        # crawl4ai_folder = os.path.join(Path.home(), ".crawl4ai")
+        # crawl4ai_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
         # driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=self.options)
         # chromedriver_path = chromedriver_autoinstaller.install()
         # chromedriver_path = chromedriver_autoinstaller.utils.download_chromedriver()
         # self.service = Service(chromedriver_autoinstaller.install())
-        
-        
-        chromedriver_path = ChromeDriverManager().install()
-        self.service = Service(chromedriver_path)
-        self.service.log_path = "NUL"
-        self.driver = webdriver.Chrome(service=self.service, options=self.options)
-        self.driver = self.execute_hook('on_driver_created', self.driver)
-        
+
+        # chromedriver_path = ChromeDriverManager().install()
+        # self.service = Service(chromedriver_path)
+        # self.service.log_path = "NUL"
+        # self.driver = webdriver.Chrome(service=self.service, options=self.options)
+
+        # Use selenium-manager (built into Selenium 4.10.0+)
+        self.service = Service()
+        self.driver = webdriver.Chrome(options=self.options)
+
+        self.driver = self.execute_hook("on_driver_created", self.driver)
+
         if kwargs.get("cookies"):
             for cookie in kwargs.get("cookies"):
                 self.driver.add_cookie(cookie)
-            
-        
 
     def set_hook(self, hook_type: str, hook: Callable):
         if hook_type in self.hooks:
             self.hooks[hook_type] = hook
         else:
             raise ValueError(f"Invalid hook type: {hook_type}")
-    
+
     def execute_hook(self, hook_type: str, *args):
         hook = self.hooks.get(hook_type)
         if hook:
@@ -161,7 +172,9 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
                 if isinstance(result, webdriver.Chrome):
                     return result
                 else:
-                    raise TypeError(f"Hook {hook_type} must return an instance of webdriver.Chrome or None.")
+                    raise TypeError(
+                        f"Hook {hook_type} must return an instance of webdriver.Chrome or None."
+                    )
         # If the hook returns None or there is no hook, return self.driver
         return self.driver
 
@@ -169,96 +182,158 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         self.options.add_argument(f"user-agent={user_agent}")
         self.driver.quit()
         self.driver = webdriver.Chrome(service=self.service, options=self.options)
-        self.driver = self.execute_hook('on_user_agent_updated', self.driver)
+        self.driver = self.execute_hook("on_user_agent_updated", self.driver)
 
     def set_custom_headers(self, headers: dict):
         # Enable Network domain for sending headers
-        self.driver.execute_cdp_cmd('Network.enable', {})
+        self.driver.execute_cdp_cmd("Network.enable", {})
         # Set extra HTTP headers
-        self.driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', {'headers': headers})
+        self.driver.execute_cdp_cmd("Network.setExtraHTTPHeaders", {"headers": headers})
 
+    def _ensure_page_load(self, max_checks=6, check_interval=0.01):
+        initial_length = len(self.driver.page_source)
 
-    def crawl(self, url: str) -> str:
+        for ix in range(max_checks):
+            # print(f"Checking page load: {ix}")
+            time.sleep(check_interval)
+            current_length = len(self.driver.page_source)
+
+            if current_length != initial_length:
+                break
+
+        return self.driver.page_source
+
+    def crawl(self, url: str, **kwargs) -> str:
         # Create md5 hash of the URL
         import hashlib
+
         url_hash = hashlib.md5(url.encode()).hexdigest()
-        
+
         if self.use_cached_html:
-            cache_file_path = os.path.join(Path.home(), ".crawl4ai", "cache", url_hash)
+            cache_file_path = os.path.join(
+                os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()),
+                ".crawl4ai",
+                "cache",
+                url_hash,
+            )
             if os.path.exists(cache_file_path):
                 with open(cache_file_path, "r") as f:
-                    return f.read()
+                    return sanitize_input_encode(f.read())
 
         try:
-            self.driver = self.execute_hook('before_get_url', self.driver)
+            self.driver = self.execute_hook("before_get_url", self.driver)
             if self.verbose:
                 print(f"[LOG] 🕸️ Crawling {url} using LocalSeleniumCrawlerStrategy...")
-            self.driver.get(url) #<html><head></head><body></body></html>
-            html = self.driver.page_source                
+            self.driver.get(url)  # <html><head></head><body></body></html>
+
+            WebDriverWait(self.driver, 20).until(
+                lambda d: d.execute_script("return document.readyState") == "complete"
+            )
             WebDriverWait(self.driver, 10).until(
                 EC.presence_of_all_elements_located((By.TAG_NAME, "body"))
             )
-            can_not_be_done_headless = False # Look at my creativity for naming variables
-            # TODO: Very ugly way for now but it works
-            if html == "<html><head></head><body></body></html>":
+
+            self.driver.execute_script(
+                "window.scrollTo(0, document.body.scrollHeight);"
+            )
+
+            self.driver = self.execute_hook("after_get_url", self.driver)
+            html = sanitize_input_encode(
+                self._ensure_page_load()
+            )  # self.driver.page_source
+            can_not_be_done_headless = (
+                False  # Look at my creativity for naming variables
+            )
+
+            # TODO: Very ugly approach, but promise to change it!
+            if (
+                kwargs.get("bypass_headless", False)
+                or html == "<html><head></head><body></body></html>"
+            ):
+                print(
+                    "[LOG] 🙌 Page could not be loaded in headless mode. Trying non-headless mode..."
+                )
                 can_not_be_done_headless = True
                 options = Options()
                 options.headless = False
                 # set window size very small
-                options.add_argument("--window-size=10,10")
+                options.add_argument("--window-size=5,5")
                 driver = webdriver.Chrome(service=self.service, options=options)
                 driver.get(url)
-                html = driver.page_source
+                self.driver = self.execute_hook("after_get_url", driver)
+                html = sanitize_input_encode(driver.page_source)
                 driver.quit()
-            
-            self.driver = self.execute_hook('after_get_url', self.driver)
-            
+
             # Execute JS code if provided
+            self.js_code = kwargs.get("js_code", self.js_code)
             if self.js_code and type(self.js_code) == str:
                 self.driver.execute_script(self.js_code)
                 # Optionally, wait for some condition after executing the JS code
                 WebDriverWait(self.driver, 10).until(
-                    lambda driver: driver.execute_script("return document.readyState") == "complete"
+                    lambda driver: driver.execute_script("return document.readyState")
+                    == "complete"
                 )
             elif self.js_code and type(self.js_code) == list:
                 for js in self.js_code:
                     self.driver.execute_script(js)
                     WebDriverWait(self.driver, 10).until(
-                        lambda driver: driver.execute_script("return document.readyState") == "complete"
+                        lambda driver: driver.execute_script(
+                            "return document.readyState"
+                        )
+                        == "complete"
                     )
-            
+
+            # Optionally, wait for some condition after executing the JS code : Contributed by (https://github.com/jonymusky)
+            wait_for = kwargs.get("wait_for", False)
+            if wait_for:
+                if callable(wait_for):
+                    print("[LOG] 🔄 Waiting for condition...")
+                    WebDriverWait(self.driver, 20).until(wait_for)
+                else:
+                    print("[LOG] 🔄 Waiting for condition...")
+                    WebDriverWait(self.driver, 20).until(
+                        EC.presence_of_element_located((By.CSS_SELECTOR, wait_for))
+                    )
+
             if not can_not_be_done_headless:
-                html = self.driver.page_source
-            self.driver = self.execute_hook('before_return_html', self.driver, html)
-            
+                html = sanitize_input_encode(self.driver.page_source)
+            self.driver = self.execute_hook("before_return_html", self.driver, html)
+
             # Store in cache
-            cache_file_path = os.path.join(Path.home(), ".crawl4ai", "cache", url_hash)
-            with open(cache_file_path, "w") as f:
+            cache_file_path = os.path.join(
+                os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()),
+                ".crawl4ai",
+                "cache",
+                url_hash,
+            )
+            with open(cache_file_path, "w", encoding="utf-8") as f:
                 f.write(html)
-                
+
             if self.verbose:
                 print(f"[LOG] ✅ Crawled {url} successfully!")
-            
+
             return html
-        except InvalidArgumentException:
-            if not hasattr(e, 'msg'):
-                e.msg = str(e)
+        except InvalidArgumentException as e:
+            if not hasattr(e, "msg"):
+                e.msg = sanitize_input_encode(str(e))
             raise InvalidArgumentException(f"Failed to crawl {url}: {e.msg}")
         except WebDriverException as e:
             # If e does nlt have msg attribute create it and set it to str(e)
-            if not hasattr(e, 'msg'):
-                e.msg = str(e)
-            raise WebDriverException(f"Failed to crawl {url}: {e.msg}")  
+            if not hasattr(e, "msg"):
+                e.msg = sanitize_input_encode(str(e))
+            raise WebDriverException(f"Failed to crawl {url}: {e.msg}")
         except Exception as e:
-            if not hasattr(e, 'msg'):
-                e.msg = str(e)
+            if not hasattr(e, "msg"):
+                e.msg = sanitize_input_encode(str(e))
             raise Exception(f"Failed to crawl {url}: {e.msg}")
 
     def take_screenshot(self) -> str:
         try:
             # Get the dimensions of the page
             total_width = self.driver.execute_script("return document.body.scrollWidth")
-            total_height = self.driver.execute_script("return document.body.scrollHeight")
+            total_height = self.driver.execute_script(
+                "return document.body.scrollHeight"
+            )
 
             # Set the window size to the dimensions of the page
             self.driver.set_window_size(total_width, total_height)
@@ -269,29 +344,33 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
             # Open the screenshot with PIL
             image = Image.open(BytesIO(screenshot))
 
+            # Convert image to RGB mode (this will handle both RGB and RGBA images)
+            rgb_image = image.convert("RGB")
+
             # Convert to JPEG and compress
             buffered = BytesIO()
-            image.save(buffered, format="JPEG", quality=85)
-            img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
+            rgb_image.save(buffered, format="JPEG", quality=85)
+            img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
 
             if self.verbose:
-                print(f"[LOG] 📸 Screenshot taken and converted to base64")
+                print("[LOG] 📸 Screenshot taken and converted to base64")
 
             return img_base64
-
         except Exception as e:
-            error_message = f"Failed to take screenshot: {str(e)}"
+            error_message = sanitize_input_encode(
+                f"Failed to take screenshot: {str(e)}"
+            )
             print(error_message)
 
             # Generate an image with black background
-            img = Image.new('RGB', (800, 600), color='black')
+            img = Image.new("RGB", (800, 600), color="black")
             draw = ImageDraw.Draw(img)
-            
+
             # Load a font
             try:
                 font = ImageFont.truetype("arial.ttf", 40)
             except IOError:
-                font = ImageFont.load_default(size=40)
+                font = ImageFont.load_default()
 
             # Define text color and wrap the text
             text_color = (255, 255, 255)
@@ -300,16 +379,16 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
 
             # Calculate text position
             text_position = (10, 10)
-            
+
             # Draw the text on the image
             draw.text(text_position, wrapped_text, fill=text_color, font=font)
-            
+
             # Convert to base64
             buffered = BytesIO()
             img.save(buffered, format="JPEG")
-            img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
+            img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
 
             return img_base64
 
     def quit(self):
-        self.driver.quit()
+        self.driver.quit()

crawl4ai/legacy/database.py

@@ -3,15 +3,17 @@ from pathlib import Path
 import sqlite3
 from typing import Optional, Tuple
 
-DB_PATH = os.path.join(Path.home(), ".crawl4ai")
+DB_PATH = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
 os.makedirs(DB_PATH, exist_ok=True)
 DB_PATH = os.path.join(DB_PATH, "crawl4ai.db")
 
+
 def init_db():
     global DB_PATH
     conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
-    cursor.execute('''
+    cursor.execute(
+        """
         CREATE TABLE IF NOT EXISTS crawled_data (
             url TEXT PRIMARY KEY,
             html TEXT,
@@ -20,35 +22,46 @@ def init_db():
             extracted_content TEXT,
             success BOOLEAN,
             media TEXT DEFAULT "{}",
-            link TEXT DEFAULT "{}",
+            links TEXT DEFAULT "{}",
             metadata TEXT DEFAULT "{}",
             screenshot TEXT DEFAULT ""
         )
-    ''')
+    """
+    )
     conn.commit()
     conn.close()
 
+
 def alter_db_add_screenshot(new_column: str = "media"):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""')
+        cursor.execute(
+            f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""'
+        )
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error altering database to add screenshot column: {e}")
 
+
 def check_db_path():
     if not DB_PATH:
         raise ValueError("Database path is not set or is empty.")
 
-def get_cached_url(url: str) -> Optional[Tuple[str, str, str, str, str, str, str, bool, str]]:
+
+def get_cached_url(
+    url: str,
+) -> Optional[Tuple[str, str, str, str, str, str, str, bool, str]]:
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('SELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?', (url,))
+        cursor.execute(
+            "SELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?",
+            (url,),
+        )
         result = cursor.fetchone()
         conn.close()
         return result
@@ -56,12 +69,25 @@ def get_cached_url(url: str) -> Optional[Tuple[str, str, str, str, str, str, str
         print(f"Error retrieving cached URL: {e}")
         return None
 
-def cache_url(url: str, html: str, cleaned_html: str, markdown: str, extracted_content: str, success: bool, media : str = "{}", links : str = "{}", metadata : str = "{}", screenshot: str = ""):
+
+def cache_url(
+    url: str,
+    html: str,
+    cleaned_html: str,
+    markdown: str,
+    extracted_content: str,
+    success: bool,
+    media: str = "{}",
+    links: str = "{}",
+    metadata: str = "{}",
+    screenshot: str = "",
+):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('''
+        cursor.execute(
+            """
             INSERT INTO crawled_data (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ON CONFLICT(url) DO UPDATE SET
@@ -74,18 +100,32 @@ def cache_url(url: str, html: str, cleaned_html: str, markdown: str, extracted_c
                 links = excluded.links,    
                 metadata = excluded.metadata,      
                 screenshot = excluded.screenshot
-        ''', (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot))
+        """,
+            (
+                url,
+                html,
+                cleaned_html,
+                markdown,
+                extracted_content,
+                success,
+                media,
+                links,
+                metadata,
+                screenshot,
+            ),
+        )
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error caching URL: {e}")
 
+
 def get_total_count() -> int:
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('SELECT COUNT(*) FROM crawled_data')
+        cursor.execute("SELECT COUNT(*) FROM crawled_data")
         result = cursor.fetchone()
         conn.close()
         return result[0]
@@ -93,40 +133,48 @@ def get_total_count() -> int:
         print(f"Error getting total count: {e}")
         return 0
 
+
 def clear_db():
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('DELETE FROM crawled_data')
+        cursor.execute("DELETE FROM crawled_data")
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error clearing database: {e}")
-        
+
+
 def flush_db():
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('DROP TABLE crawled_data')
+        cursor.execute("DROP TABLE crawled_data")
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error flushing database: {e}")
 
+
 def update_existing_records(new_column: str = "media", default_value: str = "{}"):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute(f'UPDATE crawled_data SET {new_column} = "{default_value}" WHERE screenshot IS NULL')
+        cursor.execute(
+            f'UPDATE crawled_data SET {new_column} = "{default_value}" WHERE screenshot IS NULL'
+        )
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error updating existing records: {e}")
 
+
 if __name__ == "__main__":
-    init_db()  # Initialize the database if not already initialized
-    alter_db_add_screenshot("metadata")  # Add the new column to the table
-    update_existing_records("metadata")  # Update existing records to set the new column to an empty string
+    # Delete the existing database file
+    if os.path.exists(DB_PATH):
+        os.remove(DB_PATH)
+    init_db()
+    # alter_db_add_screenshot("COL_NAME")

crawl4ai/legacy/docs_manager.py

@@ -0,0 +1,75 @@
+import requests
+import shutil
+from pathlib import Path
+from crawl4ai.async_logger import AsyncLogger
+from crawl4ai.llmtxt import AsyncLLMTextManager
+
+
+class DocsManager:
+    def __init__(self, logger=None):
+        self.docs_dir = Path.home() / ".crawl4ai" / "docs"
+        self.local_docs = Path(__file__).parent.parent / "docs" / "llm.txt"
+        self.docs_dir.mkdir(parents=True, exist_ok=True)
+        self.logger = logger or AsyncLogger(verbose=True)
+        self.llm_text = AsyncLLMTextManager(self.docs_dir, self.logger)
+
+    async def ensure_docs_exist(self):
+        """Fetch docs if not present"""
+        if not any(self.docs_dir.iterdir()):
+            await self.fetch_docs()
+
+    async def fetch_docs(self) -> bool:
+        """Copy from local docs or download from GitHub"""
+        try:
+            # Try local first
+            if self.local_docs.exists() and (
+                any(self.local_docs.glob("*.md"))
+                or any(self.local_docs.glob("*.tokens"))
+            ):
+                # Empty the local docs directory
+                for file_path in self.docs_dir.glob("*.md"):
+                    file_path.unlink()
+                # for file_path in self.docs_dir.glob("*.tokens"):
+                #     file_path.unlink()
+                for file_path in self.local_docs.glob("*.md"):
+                    shutil.copy2(file_path, self.docs_dir / file_path.name)
+                # for file_path in self.local_docs.glob("*.tokens"):
+                #     shutil.copy2(file_path, self.docs_dir / file_path.name)
+                return True
+
+            # Fallback to GitHub
+            response = requests.get(
+                "https://api.github.com/repos/unclecode/crawl4ai/contents/docs/llm.txt",
+                headers={"Accept": "application/vnd.github.v3+json"},
+            )
+            response.raise_for_status()
+
+            for item in response.json():
+                if item["type"] == "file" and item["name"].endswith(".md"):
+                    content = requests.get(item["download_url"]).text
+                    with open(self.docs_dir / item["name"], "w", encoding="utf-8") as f:
+                        f.write(content)
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Failed to fetch docs: {str(e)}")
+            raise
+
+    def list(self) -> list[str]:
+        """List available topics"""
+        names = [file_path.stem for file_path in self.docs_dir.glob("*.md")]
+        # Remove [0-9]+_ prefix
+        names = [name.split("_", 1)[1] if name[0].isdigit() else name for name in names]
+        # Exclude those end with .xs.md and .q.md
+        names = [
+            name
+            for name in names
+            if not name.endswith(".xs") and not name.endswith(".q")
+        ]
+        return names
+
+    def generate(self, sections, mode="extended"):
+        return self.llm_text.generate(sections, mode)
+
+    def search(self, query: str, top_k: int = 5):
+        return self.llm_text.search(query, top_k)

crawl4ai/legacy/llmtxt.py

@@ -0,0 +1,546 @@
+import os
+from pathlib import Path
+import re
+from typing import Dict, List, Tuple, Optional, Any
+import json
+from tqdm import tqdm
+import time
+import psutil
+import numpy as np
+from rank_bm25 import BM25Okapi
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from litellm import batch_completion
+from .async_logger import AsyncLogger
+import litellm
+import pickle
+import hashlib  # <--- ADDED for file-hash
+import glob
+
+litellm.set_verbose = False
+
+
+def _compute_file_hash(file_path: Path) -> str:
+    """Compute MD5 hash for the file's entire content."""
+    hash_md5 = hashlib.md5()
+    with file_path.open("rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+class AsyncLLMTextManager:
+    def __init__(
+        self,
+        docs_dir: Path,
+        logger: Optional[AsyncLogger] = None,
+        max_concurrent_calls: int = 5,
+        batch_size: int = 3,
+    ) -> None:
+        self.docs_dir = docs_dir
+        self.logger = logger
+        self.max_concurrent_calls = max_concurrent_calls
+        self.batch_size = batch_size
+        self.bm25_index = None
+        self.document_map: Dict[str, Any] = {}
+        self.tokenized_facts: List[str] = []
+        self.bm25_index_file = self.docs_dir / "bm25_index.pkl"
+
+    async def _process_document_batch(self, doc_batch: List[Path]) -> None:
+        """Process a batch of documents in parallel"""
+        contents = []
+        for file_path in doc_batch:
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    contents.append(f.read())
+            except Exception as e:
+                self.logger.error(f"Error reading {file_path}: {str(e)}")
+                contents.append("")  # Add empty content to maintain batch alignment
+
+        prompt = """Given a documentation file, generate a list of atomic facts where each fact:
+1. Represents a single piece of knowledge
+2. Contains variations in terminology for the same concept
+3. References relevant code patterns if they exist
+4. Is written in a way that would match natural language queries
+
+Each fact should follow this format:
+<main_concept>: <fact_statement> | <related_terms> | <code_reference>
+
+Example Facts:
+browser_config: Configure headless mode and browser type for AsyncWebCrawler | headless, browser_type, chromium, firefox | BrowserConfig(browser_type="chromium", headless=True)
+redis_connection: Redis client connection requires host and port configuration | redis setup, redis client, connection params | Redis(host='localhost', port=6379, db=0)
+pandas_filtering: Filter DataFrame rows using boolean conditions | dataframe filter, query, boolean indexing | df[df['column'] > 5]
+
+Wrap your response in <index>...</index> tags.
+"""
+
+        # Prepare messages for batch processing
+        messages_list = [
+            [
+                {
+                    "role": "user",
+                    "content": f"{prompt}\n\nGenerate index for this documentation:\n\n{content}",
+                }
+            ]
+            for content in contents
+            if content
+        ]
+
+        try:
+            responses = batch_completion(
+                model="anthropic/claude-3-5-sonnet-latest",
+                messages=messages_list,
+                logger_fn=None,
+            )
+
+            # Process responses and save index files
+            for response, file_path in zip(responses, doc_batch):
+                try:
+                    index_content_match = re.search(
+                        r"<index>(.*?)</index>",
+                        response.choices[0].message.content,
+                        re.DOTALL,
+                    )
+                    if not index_content_match:
+                        self.logger.warning(
+                            f"No <index>...</index> content found for {file_path}"
+                        )
+                        continue
+
+                    index_content = re.sub(
+                        r"\n\s*\n", "\n", index_content_match.group(1)
+                    ).strip()
+                    if index_content:
+                        index_file = file_path.with_suffix(".q.md")
+                        with open(index_file, "w", encoding="utf-8") as f:
+                            f.write(index_content)
+                        self.logger.info(f"Created index file: {index_file}")
+                    else:
+                        self.logger.warning(
+                            f"No index content found in response for {file_path}"
+                        )
+
+                except Exception as e:
+                    self.logger.error(
+                        f"Error processing response for {file_path}: {str(e)}"
+                    )
+
+        except Exception as e:
+            self.logger.error(f"Error in batch completion: {str(e)}")
+
+    def _validate_fact_line(self, line: str) -> Tuple[bool, Optional[str]]:
+        if "|" not in line:
+            return False, "Missing separator '|'"
+
+        parts = [p.strip() for p in line.split("|")]
+        if len(parts) != 3:
+            return False, f"Expected 3 parts, got {len(parts)}"
+
+        concept_part = parts[0]
+        if ":" not in concept_part:
+            return False, "Missing ':' in concept definition"
+
+        return True, None
+
+    def _load_or_create_token_cache(self, fact_file: Path) -> Dict:
+        """
+        Load token cache from .q.tokens if present and matching file hash.
+        Otherwise return a new structure with updated file-hash.
+        """
+        cache_file = fact_file.with_suffix(".q.tokens")
+        current_hash = _compute_file_hash(fact_file)
+
+        if cache_file.exists():
+            try:
+                with open(cache_file, "r") as f:
+                    cache = json.load(f)
+                # If the hash matches, return it directly
+                if cache.get("content_hash") == current_hash:
+                    return cache
+                # Otherwise, we signal that it's changed
+                self.logger.info(f"Hash changed for {fact_file}, reindex needed.")
+            except json.JSONDecodeError:
+                self.logger.warning(f"Corrupt token cache for {fact_file}, rebuilding.")
+            except Exception as e:
+                self.logger.warning(f"Error reading cache for {fact_file}: {str(e)}")
+
+        # Return a fresh cache
+        return {"facts": {}, "content_hash": current_hash}
+
+    def _save_token_cache(self, fact_file: Path, cache: Dict) -> None:
+        cache_file = fact_file.with_suffix(".q.tokens")
+        # Always ensure we're saving the correct file-hash
+        cache["content_hash"] = _compute_file_hash(fact_file)
+        with open(cache_file, "w") as f:
+            json.dump(cache, f)
+
+    def preprocess_text(self, text: str) -> List[str]:
+        parts = [x.strip() for x in text.split("|")] if "|" in text else [text]
+        # Remove : after the first word of parts[0]
+        parts[0] = re.sub(r"^(.*?):", r"\1", parts[0])
+
+        lemmatizer = WordNetLemmatizer()
+        stop_words = set(stopwords.words("english")) - {
+            "how",
+            "what",
+            "when",
+            "where",
+            "why",
+            "which",
+        }
+
+        tokens = []
+        for part in parts:
+            if "(" in part and ")" in part:
+                code_tokens = re.findall(
+                    r'[\w_]+(?=\()|[\w_]+(?==[\'"]{1}[\w_]+[\'"]{1})', part
+                )
+                tokens.extend(code_tokens)
+
+            words = word_tokenize(part.lower())
+            tokens.extend(
+                [
+                    lemmatizer.lemmatize(token)
+                    for token in words
+                    if token not in stop_words
+                ]
+            )
+
+        return tokens
+
+    def maybe_load_bm25_index(self, clear_cache=False) -> bool:
+        """
+        Load existing BM25 index from disk, if present and clear_cache=False.
+        """
+        if not clear_cache and os.path.exists(self.bm25_index_file):
+            self.logger.info("Loading existing BM25 index from disk.")
+            with open(self.bm25_index_file, "rb") as f:
+                data = pickle.load(f)
+            self.tokenized_facts = data["tokenized_facts"]
+            self.bm25_index = data["bm25_index"]
+            return True
+        return False
+
+    def build_search_index(self, clear_cache=False) -> None:
+        """
+        Checks for new or modified .q.md files by comparing file-hash.
+        If none need reindexing and clear_cache is False, loads existing index if available.
+        Otherwise, reindexes only changed/new files and merges or creates a new index.
+        """
+        # If clear_cache is True, we skip partial logic: rebuild everything from scratch
+        if clear_cache:
+            self.logger.info("Clearing cache and rebuilding full search index.")
+            if self.bm25_index_file.exists():
+                self.bm25_index_file.unlink()
+
+        process = psutil.Process()
+        self.logger.info("Checking which .q.md files need (re)indexing...")
+
+        # Gather all .q.md files
+        q_files = [
+            self.docs_dir / f for f in os.listdir(self.docs_dir) if f.endswith(".q.md")
+        ]
+
+        # We'll store known (unchanged) facts in these lists
+        existing_facts: List[str] = []
+        existing_tokens: List[List[str]] = []
+
+        # Keep track of invalid lines for logging
+        invalid_lines = []
+        needSet = []  # files that must be (re)indexed
+
+        for qf in q_files:
+            token_cache_file = qf.with_suffix(".q.tokens")
+
+            # If no .q.tokens or clear_cache is True → definitely reindex
+            if clear_cache or not token_cache_file.exists():
+                needSet.append(qf)
+                continue
+
+            # Otherwise, load the existing cache and compare hash
+            cache = self._load_or_create_token_cache(qf)
+            # If the .q.tokens was out of date (i.e. changed hash), we reindex
+            if len(cache["facts"]) == 0 or cache.get(
+                "content_hash"
+            ) != _compute_file_hash(qf):
+                needSet.append(qf)
+            else:
+                # File is unchanged → retrieve cached token data
+                for line, cache_data in cache["facts"].items():
+                    existing_facts.append(line)
+                    existing_tokens.append(cache_data["tokens"])
+                    self.document_map[line] = qf  # track the doc for that fact
+
+        if not needSet and not clear_cache:
+            # If no file needs reindexing, try loading existing index
+            if self.maybe_load_bm25_index(clear_cache=False):
+                self.logger.info(
+                    "No new/changed .q.md files found. Using existing BM25 index."
+                )
+                return
+            else:
+                # If there's no existing index, we must build a fresh index from the old caches
+                self.logger.info(
+                    "No existing BM25 index found. Building from cached facts."
+                )
+                if existing_facts:
+                    self.logger.info(
+                        f"Building BM25 index with {len(existing_facts)} cached facts."
+                    )
+                    self.bm25_index = BM25Okapi(existing_tokens)
+                    self.tokenized_facts = existing_facts
+                    with open(self.bm25_index_file, "wb") as f:
+                        pickle.dump(
+                            {
+                                "bm25_index": self.bm25_index,
+                                "tokenized_facts": self.tokenized_facts,
+                            },
+                            f,
+                        )
+                else:
+                    self.logger.warning("No facts found at all. Index remains empty.")
+                return
+
+        # ----------------------------------------------------- /Users/unclecode/.crawl4ai/docs/14_proxy_security.q.q.tokens '/Users/unclecode/.crawl4ai/docs/14_proxy_security.q.md'
+        # If we reach here, we have new or changed .q.md files
+        # We'll parse them, reindex them, and then combine with existing_facts
+        # -----------------------------------------------------
+
+        self.logger.info(f"{len(needSet)} file(s) need reindexing. Parsing now...")
+
+        # 1) Parse the new or changed .q.md files
+        new_facts = []
+        new_tokens = []
+        with tqdm(total=len(needSet), desc="Indexing changed files") as file_pbar:
+            for file in needSet:
+                # We'll build up a fresh cache
+                fresh_cache = {"facts": {}, "content_hash": _compute_file_hash(file)}
+                try:
+                    with open(file, "r", encoding="utf-8") as f_obj:
+                        content = f_obj.read().strip()
+                        lines = [l.strip() for l in content.split("\n") if l.strip()]
+
+                    for line in lines:
+                        is_valid, error = self._validate_fact_line(line)
+                        if not is_valid:
+                            invalid_lines.append((file, line, error))
+                            continue
+
+                        tokens = self.preprocess_text(line)
+                        fresh_cache["facts"][line] = {
+                            "tokens": tokens,
+                            "added": time.time(),
+                        }
+                        new_facts.append(line)
+                        new_tokens.append(tokens)
+                        self.document_map[line] = file
+
+                    # Save the new .q.tokens with updated hash
+                    self._save_token_cache(file, fresh_cache)
+
+                    mem_usage = process.memory_info().rss / 1024 / 1024
+                    self.logger.debug(
+                        f"Memory usage after {file.name}: {mem_usage:.2f}MB"
+                    )
+
+                except Exception as e:
+                    self.logger.error(f"Error processing {file}: {str(e)}")
+
+                file_pbar.update(1)
+
+        if invalid_lines:
+            self.logger.warning(f"Found {len(invalid_lines)} invalid fact lines:")
+            for file, line, error in invalid_lines:
+                self.logger.warning(f"{file}: {error} in line: {line[:50]}...")
+
+        # 2) Merge newly tokenized facts with the existing ones
+        all_facts = existing_facts + new_facts
+        all_tokens = existing_tokens + new_tokens
+
+        # 3) Build BM25 index from combined facts
+        self.logger.info(
+            f"Building BM25 index with {len(all_facts)} total facts (old + new)."
+        )
+        self.bm25_index = BM25Okapi(all_tokens)
+        self.tokenized_facts = all_facts
+
+        # 4) Save the updated BM25 index to disk
+        with open(self.bm25_index_file, "wb") as f:
+            pickle.dump(
+                {
+                    "bm25_index": self.bm25_index,
+                    "tokenized_facts": self.tokenized_facts,
+                },
+                f,
+            )
+
+        final_mem = process.memory_info().rss / 1024 / 1024
+        self.logger.info(f"Search index updated. Final memory usage: {final_mem:.2f}MB")
+
+    async def generate_index_files(
+        self, force_generate_facts: bool = False, clear_bm25_cache: bool = False
+    ) -> None:
+        """
+        Generate index files for all documents in parallel batches
+
+        Args:
+            force_generate_facts (bool): If True, regenerate indexes even if they exist
+            clear_bm25_cache (bool): If True, clear existing BM25 index cache
+        """
+        self.logger.info("Starting index generation for documentation files.")
+
+        md_files = [
+            self.docs_dir / f
+            for f in os.listdir(self.docs_dir)
+            if f.endswith(".md") and not any(f.endswith(x) for x in [".q.md", ".xs.md"])
+        ]
+
+        # Filter out files that already have .q files unless force=True
+        if not force_generate_facts:
+            md_files = [
+                f
+                for f in md_files
+                if not (self.docs_dir / f.name.replace(".md", ".q.md")).exists()
+            ]
+
+        if not md_files:
+            self.logger.info("All index files exist. Use force=True to regenerate.")
+        else:
+            # Process documents in batches
+            for i in range(0, len(md_files), self.batch_size):
+                batch = md_files[i : i + self.batch_size]
+                self.logger.info(
+                    f"Processing batch {i//self.batch_size + 1}/{(len(md_files)//self.batch_size) + 1}"
+                )
+                await self._process_document_batch(batch)
+
+        self.logger.info("Index generation complete, building/updating search index.")
+        self.build_search_index(clear_cache=clear_bm25_cache)
+
+    def generate(self, sections: List[str], mode: str = "extended") -> str:
+        # Get all markdown files
+        all_files = glob.glob(str(self.docs_dir / "[0-9]*.md")) + glob.glob(
+            str(self.docs_dir / "[0-9]*.xs.md")
+        )
+
+        # Extract base names without extensions
+        base_docs = {
+            Path(f).name.split(".")[0]
+            for f in all_files
+            if not Path(f).name.endswith(".q.md")
+        }
+
+        # Filter by sections if provided
+        if sections:
+            base_docs = {
+                doc
+                for doc in base_docs
+                if any(section.lower() in doc.lower() for section in sections)
+            }
+
+        # Get file paths based on mode
+        files = []
+        for doc in sorted(
+            base_docs,
+            key=lambda x: int(x.split("_")[0]) if x.split("_")[0].isdigit() else 999999,
+        ):
+            if mode == "condensed":
+                xs_file = self.docs_dir / f"{doc}.xs.md"
+                regular_file = self.docs_dir / f"{doc}.md"
+                files.append(str(xs_file if xs_file.exists() else regular_file))
+            else:
+                files.append(str(self.docs_dir / f"{doc}.md"))
+
+        # Read and format content
+        content = []
+        for file in files:
+            try:
+                with open(file, "r", encoding="utf-8") as f:
+                    fname = Path(file).name
+                    content.append(f"{'#'*20}\n# {fname}\n{'#'*20}\n\n{f.read()}")
+            except Exception as e:
+                self.logger.error(f"Error reading {file}: {str(e)}")
+
+        return "\n\n---\n\n".join(content) if content else ""
+
+    def search(self, query: str, top_k: int = 5) -> str:
+        if not self.bm25_index:
+            return "No search index available. Call build_search_index() first."
+
+        query_tokens = self.preprocess_text(query)
+        doc_scores = self.bm25_index.get_scores(query_tokens)
+
+        mean_score = np.mean(doc_scores)
+        std_score = np.std(doc_scores)
+        score_threshold = mean_score + (0.25 * std_score)
+
+        file_data = self._aggregate_search_scores(
+            doc_scores=doc_scores,
+            score_threshold=score_threshold,
+            query_tokens=query_tokens,
+        )
+
+        ranked_files = sorted(
+            file_data.items(),
+            key=lambda x: (
+                x[1]["code_match_score"] * 2.0
+                + x[1]["match_count"] * 1.5
+                + x[1]["total_score"]
+            ),
+            reverse=True,
+        )[:top_k]
+
+        results = []
+        for file, _ in ranked_files:
+            main_doc = str(file).replace(".q.md", ".md")
+            if os.path.exists(self.docs_dir / main_doc):
+                with open(self.docs_dir / main_doc, "r", encoding="utf-8") as f:
+                    only_file_name = main_doc.split("/")[-1]
+                    content = ["#" * 20, f"# {only_file_name}", "#" * 20, "", f.read()]
+                    results.append("\n".join(content))
+
+        return "\n\n---\n\n".join(results)
+
+    def _aggregate_search_scores(
+        self, doc_scores: List[float], score_threshold: float, query_tokens: List[str]
+    ) -> Dict:
+        file_data = {}
+
+        for idx, score in enumerate(doc_scores):
+            if score <= score_threshold:
+                continue
+
+            fact = self.tokenized_facts[idx]
+            file_path = self.document_map[fact]
+
+            if file_path not in file_data:
+                file_data[file_path] = {
+                    "total_score": 0,
+                    "match_count": 0,
+                    "code_match_score": 0,
+                    "matched_facts": [],
+                }
+
+            components = fact.split("|") if "|" in fact else [fact]
+
+            code_match_score = 0
+            if len(components) == 3:
+                code_ref = components[2].strip()
+                code_tokens = self.preprocess_text(code_ref)
+                code_match_score = len(set(query_tokens) & set(code_tokens)) / len(
+                    query_tokens
+                )
+
+            file_data[file_path]["total_score"] += score
+            file_data[file_path]["match_count"] += 1
+            file_data[file_path]["code_match_score"] = max(
+                file_data[file_path]["code_match_score"], code_match_score
+            )
+            file_data[file_path]["matched_facts"].append(fact)
+
+        return file_data
+
+    def refresh_index(self) -> None:
+        """Convenience method for a full rebuild."""
+        self.build_search_index(clear_cache=True)

crawl4ai/legacy/version_manager.py

@@ -0,0 +1,29 @@
+# version_manager.py
+from pathlib import Path
+from packaging import version
+from . import __version__
+
+
+class VersionManager:
+    def __init__(self):
+        self.home_dir = Path.home() / ".crawl4ai"
+        self.version_file = self.home_dir / "version.txt"
+
+    def get_installed_version(self):
+        """Get the version recorded in home directory"""
+        if not self.version_file.exists():
+            return None
+        try:
+            return version.parse(self.version_file.read_text().strip())
+        except:
+            return None
+
+    def update_version(self):
+        """Update the version file to current library version"""
+        self.version_file.write_text(__version__.__version__)
+
+    def needs_update(self):
+        """Check if database needs update based on version"""
+        installed = self.get_installed_version()
+        current = version.parse(__version__.__version__)
+        return installed is None or installed < current

crawl4ai/legacy/web_crawler.py

@@ -0,0 +1,294 @@
+import os, time
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+from pathlib import Path
+
+from .models import UrlModel, CrawlResult
+from .database import init_db, get_cached_url, cache_url
+from .utils import *
+from .chunking_strategy import *
+from .extraction_strategy import *
+from .crawler_strategy import *
+from typing import List
+from concurrent.futures import ThreadPoolExecutor
+from .content_scraping_strategy import WebScrapingStrategy
+from .config import *
+import warnings
+import json
+
+warnings.filterwarnings(
+    "ignore",
+    message='Field "model_name" has conflict with protected namespace "model_".',
+)
+
+
+class WebCrawler:
+    def __init__(
+        self,
+        crawler_strategy: CrawlerStrategy = None,
+        always_by_pass_cache: bool = False,
+        verbose: bool = False,
+    ):
+        self.crawler_strategy = crawler_strategy or LocalSeleniumCrawlerStrategy(
+            verbose=verbose
+        )
+        self.always_by_pass_cache = always_by_pass_cache
+        self.crawl4ai_folder = os.path.join(
+            os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
+        )
+        os.makedirs(self.crawl4ai_folder, exist_ok=True)
+        os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True)
+        init_db()
+        self.ready = False
+
+    def warmup(self):
+        print("[LOG] 🌤️  Warming up the WebCrawler")
+        self.run(
+            url="https://google.com/",
+            word_count_threshold=5,
+            extraction_strategy=NoExtractionStrategy(),
+            bypass_cache=False,
+            verbose=False,
+        )
+        self.ready = True
+        print("[LOG] 🌞 WebCrawler is ready to crawl")
+
+    def fetch_page(
+        self,
+        url_model: UrlModel,
+        provider: str = DEFAULT_PROVIDER,
+        api_token: str = None,
+        extract_blocks_flag: bool = True,
+        word_count_threshold=MIN_WORD_THRESHOLD,
+        css_selector: str = None,
+        screenshot: bool = False,
+        use_cached_html: bool = False,
+        extraction_strategy: ExtractionStrategy = None,
+        chunking_strategy: ChunkingStrategy = RegexChunking(),
+        **kwargs,
+    ) -> CrawlResult:
+        return self.run(
+            url_model.url,
+            word_count_threshold,
+            extraction_strategy or NoExtractionStrategy(),
+            chunking_strategy,
+            bypass_cache=url_model.forced,
+            css_selector=css_selector,
+            screenshot=screenshot,
+            **kwargs,
+        )
+        pass
+
+    def fetch_pages(
+        self,
+        url_models: List[UrlModel],
+        provider: str = DEFAULT_PROVIDER,
+        api_token: str = None,
+        extract_blocks_flag: bool = True,
+        word_count_threshold=MIN_WORD_THRESHOLD,
+        use_cached_html: bool = False,
+        css_selector: str = None,
+        screenshot: bool = False,
+        extraction_strategy: ExtractionStrategy = None,
+        chunking_strategy: ChunkingStrategy = RegexChunking(),
+        **kwargs,
+    ) -> List[CrawlResult]:
+        extraction_strategy = extraction_strategy or NoExtractionStrategy()
+
+        def fetch_page_wrapper(url_model, *args, **kwargs):
+            return self.fetch_page(url_model, *args, **kwargs)
+
+        with ThreadPoolExecutor() as executor:
+            results = list(
+                executor.map(
+                    fetch_page_wrapper,
+                    url_models,
+                    [provider] * len(url_models),
+                    [api_token] * len(url_models),
+                    [extract_blocks_flag] * len(url_models),
+                    [word_count_threshold] * len(url_models),
+                    [css_selector] * len(url_models),
+                    [screenshot] * len(url_models),
+                    [use_cached_html] * len(url_models),
+                    [extraction_strategy] * len(url_models),
+                    [chunking_strategy] * len(url_models),
+                    *[kwargs] * len(url_models),
+                )
+            )
+
+        return results
+
+    def run(
+        self,
+        url: str,
+        word_count_threshold=MIN_WORD_THRESHOLD,
+        extraction_strategy: ExtractionStrategy = None,
+        chunking_strategy: ChunkingStrategy = RegexChunking(),
+        bypass_cache: bool = False,
+        css_selector: str = None,
+        screenshot: bool = False,
+        user_agent: str = None,
+        verbose=True,
+        **kwargs,
+    ) -> CrawlResult:
+        try:
+            extraction_strategy = extraction_strategy or NoExtractionStrategy()
+            extraction_strategy.verbose = verbose
+            if not isinstance(extraction_strategy, ExtractionStrategy):
+                raise ValueError("Unsupported extraction strategy")
+            if not isinstance(chunking_strategy, ChunkingStrategy):
+                raise ValueError("Unsupported chunking strategy")
+
+            word_count_threshold = max(word_count_threshold, MIN_WORD_THRESHOLD)
+
+            cached = None
+            screenshot_data = None
+            extracted_content = None
+            if not bypass_cache and not self.always_by_pass_cache:
+                cached = get_cached_url(url)
+
+            if kwargs.get("warmup", True) and not self.ready:
+                return None
+
+            if cached:
+                html = sanitize_input_encode(cached[1])
+                extracted_content = sanitize_input_encode(cached[4])
+                if screenshot:
+                    screenshot_data = cached[9]
+                    if not screenshot_data:
+                        cached = None
+
+            if not cached or not html:
+                if user_agent:
+                    self.crawler_strategy.update_user_agent(user_agent)
+                t1 = time.time()
+                html = sanitize_input_encode(self.crawler_strategy.crawl(url, **kwargs))
+                t2 = time.time()
+                if verbose:
+                    print(
+                        f"[LOG] 🚀 Crawling done for {url}, success: {bool(html)}, time taken: {t2 - t1:.2f} seconds"
+                    )
+                if screenshot:
+                    screenshot_data = self.crawler_strategy.take_screenshot()
+
+            crawl_result = self.process_html(
+                url,
+                html,
+                extracted_content,
+                word_count_threshold,
+                extraction_strategy,
+                chunking_strategy,
+                css_selector,
+                screenshot_data,
+                verbose,
+                bool(cached),
+                **kwargs,
+            )
+            crawl_result.success = bool(html)
+            return crawl_result
+        except Exception as e:
+            if not hasattr(e, "msg"):
+                e.msg = str(e)
+            print(f"[ERROR] 🚫 Failed to crawl {url}, error: {e.msg}")
+            return CrawlResult(url=url, html="", success=False, error_message=e.msg)
+
+    def process_html(
+        self,
+        url: str,
+        html: str,
+        extracted_content: str,
+        word_count_threshold: int,
+        extraction_strategy: ExtractionStrategy,
+        chunking_strategy: ChunkingStrategy,
+        css_selector: str,
+        screenshot: bool,
+        verbose: bool,
+        is_cached: bool,
+        **kwargs,
+    ) -> CrawlResult:
+        t = time.time()
+        # Extract content from HTML
+        try:
+            t1 = time.time()
+            scrapping_strategy = WebScrapingStrategy()
+            extra_params = {
+                k: v
+                for k, v in kwargs.items()
+                if k not in ["only_text", "image_description_min_word_threshold"]
+            }
+            result = scrapping_strategy.scrap(
+                url,
+                html,
+                word_count_threshold=word_count_threshold,
+                css_selector=css_selector,
+                only_text=kwargs.get("only_text", False),
+                image_description_min_word_threshold=kwargs.get(
+                    "image_description_min_word_threshold",
+                    IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
+                ),
+                **extra_params,
+            )
+
+            # result = get_content_of_website_optimized(url, html, word_count_threshold, css_selector=css_selector, only_text=kwargs.get("only_text", False))
+            if verbose:
+                print(
+                    f"[LOG] 🚀 Content extracted for {url}, success: True, time taken: {time.time() - t1:.2f} seconds"
+                )
+
+            if result is None:
+                raise ValueError(f"Failed to extract content from the website: {url}")
+        except InvalidCSSSelectorError as e:
+            raise ValueError(str(e))
+
+        cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
+        markdown = sanitize_input_encode(result.get("markdown", ""))
+        media = result.get("media", [])
+        links = result.get("links", [])
+        metadata = result.get("metadata", {})
+
+        if extracted_content is None:
+            if verbose:
+                print(
+                    f"[LOG] 🔥 Extracting semantic blocks for {url}, Strategy: {extraction_strategy.name}"
+                )
+
+            sections = chunking_strategy.chunk(markdown)
+            extracted_content = extraction_strategy.run(url, sections)
+            extracted_content = json.dumps(
+                extracted_content, indent=4, default=str, ensure_ascii=False
+            )
+
+            if verbose:
+                print(
+                    f"[LOG] 🚀 Extraction done for {url}, time taken: {time.time() - t:.2f} seconds."
+                )
+
+        screenshot = None if not screenshot else screenshot
+
+        if not is_cached:
+            cache_url(
+                url,
+                html,
+                cleaned_html,
+                markdown,
+                extracted_content,
+                True,
+                json.dumps(media),
+                json.dumps(links),
+                json.dumps(metadata),
+                screenshot=screenshot,
+            )
+
+        return CrawlResult(
+            url=url,
+            html=html,
+            cleaned_html=format_html(cleaned_html),
+            markdown=markdown,
+            media=media,
+            links=links,
+            metadata=metadata,
+            screenshot=screenshot,
+            extracted_content=extracted_content,
+            success=True,
+            error_message="",
+        )

crawl4ai/markdown_generation_strategy.py

@@ -0,0 +1,256 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any, Tuple
+from .models import MarkdownGenerationResult
+from .html2text import CustomHTML2Text
+# from .types import RelevantContentFilter
+from .content_filter_strategy import RelevantContentFilter
+import re
+from urllib.parse import urljoin
+
+# Pre-compile the regex pattern
+LINK_PATTERN = re.compile(r'!?\[([^\]]+)\]\(([^)]+?)(?:\s+"([^"]*)")?\)')
+
+
+def fast_urljoin(base: str, url: str) -> str:
+    """Fast URL joining for common cases."""
+    if url.startswith(("http://", "https://", "mailto:", "//")):
+        return url
+    if url.startswith("/"):
+        # Handle absolute paths
+        if base.endswith("/"):
+            return base[:-1] + url
+        return base + url
+    return urljoin(base, url)
+
+
+class MarkdownGenerationStrategy(ABC):
+    """Abstract base class for markdown generation strategies."""
+
+    def __init__(
+        self,
+        content_filter: Optional[RelevantContentFilter] = None,
+        options: Optional[Dict[str, Any]] = None,
+        verbose: bool = False,
+    ):
+        self.content_filter = content_filter
+        self.options = options or {}
+        self.verbose = verbose
+
+    @abstractmethod
+    def generate_markdown(
+        self,
+        cleaned_html: str,
+        base_url: str = "",
+        html2text_options: Optional[Dict[str, Any]] = None,
+        content_filter: Optional[RelevantContentFilter] = None,
+        citations: bool = True,
+        **kwargs,
+    ) -> MarkdownGenerationResult:
+        """Generate markdown from cleaned HTML."""
+        pass
+
+
+class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
+    """
+    Default implementation of markdown generation strategy.
+
+    How it works:
+    1. Generate raw markdown from cleaned HTML.
+    2. Convert links to citations.
+    3. Generate fit markdown if content filter is provided.
+    4. Return MarkdownGenerationResult.
+
+    Args:
+        content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
+        options (Optional[Dict[str, Any]]): Additional options for markdown generation. Defaults to None.
+
+    Returns:
+        MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
+    """
+
+    def __init__(
+        self,
+        content_filter: Optional[RelevantContentFilter] = None,
+        options: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__(content_filter, options)
+
+    def convert_links_to_citations(
+        self, markdown: str, base_url: str = ""
+    ) -> Tuple[str, str]:
+        """
+        Convert links in markdown to citations.
+
+        How it works:
+        1. Find all links in the markdown.
+        2. Convert links to citations.
+        3. Return converted markdown and references markdown.
+
+        Note:
+        This function uses a regex pattern to find links in markdown.
+
+        Args:
+            markdown (str): Markdown text.
+            base_url (str): Base URL for URL joins.
+
+        Returns:
+            Tuple[str, str]: Converted markdown and references markdown.
+        """
+        link_map = {}
+        url_cache = {}  # Cache for URL joins
+        parts = []
+        last_end = 0
+        counter = 1
+
+        for match in LINK_PATTERN.finditer(markdown):
+            parts.append(markdown[last_end : match.start()])
+            text, url, title = match.groups()
+
+            # Use cached URL if available, otherwise compute and cache
+            if base_url and not url.startswith(("http://", "https://", "mailto:")):
+                if url not in url_cache:
+                    url_cache[url] = fast_urljoin(base_url, url)
+                url = url_cache[url]
+
+            if url not in link_map:
+                desc = []
+                if title:
+                    desc.append(title)
+                if text and text != title:
+                    desc.append(text)
+                link_map[url] = (counter, ": " + " - ".join(desc) if desc else "")
+                counter += 1
+
+            num = link_map[url][0]
+            parts.append(
+                f"{text}⟨{num}⟩"
+                if not match.group(0).startswith("!")
+                else f"![{text}⟨{num}⟩]"
+            )
+            last_end = match.end()
+
+        parts.append(markdown[last_end:])
+        converted_text = "".join(parts)
+
+        # Pre-build reference strings
+        references = ["\n\n## References\n\n"]
+        references.extend(
+            f"⟨{num}⟩ {url}{desc}\n"
+            for url, (num, desc) in sorted(link_map.items(), key=lambda x: x[1][0])
+        )
+
+        return converted_text, "".join(references)
+
+    def generate_markdown(
+        self,
+        cleaned_html: str,
+        base_url: str = "",
+        html2text_options: Optional[Dict[str, Any]] = None,
+        options: Optional[Dict[str, Any]] = None,
+        content_filter: Optional[RelevantContentFilter] = None,
+        citations: bool = True,
+        **kwargs,
+    ) -> MarkdownGenerationResult:
+        """
+        Generate markdown with citations from cleaned HTML.
+
+        How it works:
+        1. Generate raw markdown from cleaned HTML.
+        2. Convert links to citations.
+        3. Generate fit markdown if content filter is provided.
+        4. Return MarkdownGenerationResult.
+
+        Args:
+            cleaned_html (str): Cleaned HTML content.
+            base_url (str): Base URL for URL joins.
+            html2text_options (Optional[Dict[str, Any]]): HTML2Text options.
+            options (Optional[Dict[str, Any]]): Additional options for markdown generation.
+            content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
+            citations (bool): Whether to generate citations.
+
+        Returns:
+            MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
+        """
+        try:
+            # Initialize HTML2Text with default options for better conversion
+            h = CustomHTML2Text(baseurl=base_url)
+            default_options = {
+                "body_width": 0,  # Disable text wrapping
+                "ignore_emphasis": False,
+                "ignore_links": False,
+                "ignore_images": False,
+                "protect_links": False,
+                "single_line_break": True,
+                "mark_code": True,
+                "escape_snob": False,
+            }
+
+            # Update with custom options if provided
+            if html2text_options:
+                default_options.update(html2text_options)
+            elif options:
+                default_options.update(options)
+            elif self.options:
+                default_options.update(self.options)
+
+            h.update_params(**default_options)
+
+            # Ensure we have valid input
+            if not cleaned_html:
+                cleaned_html = ""
+            elif not isinstance(cleaned_html, str):
+                cleaned_html = str(cleaned_html)
+
+            # Generate raw markdown
+            try:
+                raw_markdown = h.handle(cleaned_html)
+            except Exception as e:
+                raw_markdown = f"Error converting HTML to markdown: {str(e)}"
+
+            raw_markdown = raw_markdown.replace("    ```", "```")
+
+            # Convert links to citations
+            markdown_with_citations: str = raw_markdown
+            references_markdown: str = ""
+            if citations:
+                try:
+                    (
+                        markdown_with_citations,
+                        references_markdown,
+                    ) = self.convert_links_to_citations(raw_markdown, base_url)
+                except Exception as e:
+                    markdown_with_citations = raw_markdown
+                    references_markdown = f"Error generating citations: {str(e)}"
+
+            # Generate fit markdown if content filter is provided
+            fit_markdown: Optional[str] = ""
+            filtered_html: Optional[str] = ""
+            if content_filter or self.content_filter:
+                try:
+                    content_filter = content_filter or self.content_filter
+                    filtered_html = content_filter.filter_content(cleaned_html)
+                    filtered_html = "\n".join(
+                        "<div>{}</div>".format(s) for s in filtered_html
+                    )
+                    fit_markdown = h.handle(filtered_html)
+                except Exception as e:
+                    fit_markdown = f"Error generating fit markdown: {str(e)}"
+                    filtered_html = ""
+
+            return MarkdownGenerationResult(
+                raw_markdown=raw_markdown or "",
+                markdown_with_citations=markdown_with_citations or "",
+                references_markdown=references_markdown or "",
+                fit_markdown=fit_markdown or "",
+                fit_html=filtered_html or "",
+            )
+        except Exception as e:
+            # If anything fails, return empty strings with error message
+            error_msg = f"Error in markdown generation: {str(e)}"
+            return MarkdownGenerationResult(
+                raw_markdown=error_msg,
+                markdown_with_citations=error_msg,
+                references_markdown="",
+                fit_markdown="",
+                fit_html="",
+            )

crawl4ai/migrations.py

@@ -0,0 +1,194 @@
+import os
+import asyncio
+from pathlib import Path
+import aiosqlite
+from typing import Optional
+import xxhash
+import aiofiles
+import shutil
+from datetime import datetime
+from .async_logger import AsyncLogger, LogLevel
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+# logging.basicConfig(level=logging.INFO)
+# logger = logging.getLogger(__name__)
+
+
+class DatabaseMigration:
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+        self.content_paths = self._ensure_content_dirs(os.path.dirname(db_path))
+
+    def _ensure_content_dirs(self, base_path: str) -> dict:
+        dirs = {
+            "html": "html_content",
+            "cleaned": "cleaned_html",
+            "markdown": "markdown_content",
+            "extracted": "extracted_content",
+            "screenshots": "screenshots",
+        }
+        content_paths = {}
+        for key, dirname in dirs.items():
+            path = os.path.join(base_path, dirname)
+            os.makedirs(path, exist_ok=True)
+            content_paths[key] = path
+        return content_paths
+
+    def _generate_content_hash(self, content: str) -> str:
+        x = xxhash.xxh64()
+        x.update(content.encode())
+        content_hash = x.hexdigest()
+        return content_hash
+        # return hashlib.sha256(content.encode()).hexdigest()
+
+    async def _store_content(self, content: str, content_type: str) -> str:
+        if not content:
+            return ""
+
+        content_hash = self._generate_content_hash(content)
+        file_path = os.path.join(self.content_paths[content_type], content_hash)
+
+        if not os.path.exists(file_path):
+            async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
+                await f.write(content)
+
+        return content_hash
+
+    async def migrate_database(self):
+        """Migrate existing database to file-based storage"""
+        # logger.info("Starting database migration...")
+        logger.info("Starting database migration...", tag="INIT")
+
+        try:
+            async with aiosqlite.connect(self.db_path) as db:
+                # Get all rows
+                async with db.execute(
+                    """SELECT url, html, cleaned_html, markdown, 
+                       extracted_content, screenshot FROM crawled_data"""
+                ) as cursor:
+                    rows = await cursor.fetchall()
+
+                migrated_count = 0
+                for row in rows:
+                    (
+                        url,
+                        html,
+                        cleaned_html,
+                        markdown,
+                        extracted_content,
+                        screenshot,
+                    ) = row
+
+                    # Store content in files and get hashes
+                    html_hash = await self._store_content(html, "html")
+                    cleaned_hash = await self._store_content(cleaned_html, "cleaned")
+                    markdown_hash = await self._store_content(markdown, "markdown")
+                    extracted_hash = await self._store_content(
+                        extracted_content, "extracted"
+                    )
+                    screenshot_hash = await self._store_content(
+                        screenshot, "screenshots"
+                    )
+
+                    # Update database with hashes
+                    await db.execute(
+                        """
+                        UPDATE crawled_data 
+                        SET html = ?, 
+                            cleaned_html = ?,
+                            markdown = ?,
+                            extracted_content = ?,
+                            screenshot = ?
+                        WHERE url = ?
+                    """,
+                        (
+                            html_hash,
+                            cleaned_hash,
+                            markdown_hash,
+                            extracted_hash,
+                            screenshot_hash,
+                            url,
+                        ),
+                    )
+
+                    migrated_count += 1
+                    if migrated_count % 100 == 0:
+                        logger.info(f"Migrated {migrated_count} records...", tag="INIT")
+
+                await db.commit()
+                logger.success(
+                    f"Migration completed. {migrated_count} records processed.",
+                    tag="COMPLETE",
+                )
+
+        except Exception as e:
+            # logger.error(f"Migration failed: {e}")
+            logger.error(
+                message="Migration failed: {error}",
+                tag="ERROR",
+                params={"error": str(e)},
+            )
+            raise e
+
+
+async def backup_database(db_path: str) -> str:
+    """Create backup of existing database"""
+    if not os.path.exists(db_path):
+        logger.info("No existing database found. Skipping backup.", tag="INIT")
+        return None
+
+    # Create backup with timestamp
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    backup_path = f"{db_path}.backup_{timestamp}"
+
+    try:
+        # Wait for any potential write operations to finish
+        await asyncio.sleep(1)
+
+        # Create backup
+        shutil.copy2(db_path, backup_path)
+        logger.info(f"Database backup created at: {backup_path}", tag="COMPLETE")
+        return backup_path
+    except Exception as e:
+        # logger.error(f"Backup failed: {e}")
+        logger.error(
+            message="Migration failed: {error}", tag="ERROR", params={"error": str(e)}
+        )
+        raise e
+
+
+async def run_migration(db_path: Optional[str] = None):
+    """Run database migration"""
+    if db_path is None:
+        db_path = os.path.join(Path.home(), ".crawl4ai", "crawl4ai.db")
+
+    if not os.path.exists(db_path):
+        logger.info("No existing database found. Skipping migration.", tag="INIT")
+        return
+
+    # Create backup first
+    backup_path = await backup_database(db_path)
+    if not backup_path:
+        return
+
+    migration = DatabaseMigration(db_path)
+    await migration.migrate_database()
+
+
+def main():
+    """CLI entry point for migration"""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Migrate Crawl4AI database to file-based storage"
+    )
+    parser.add_argument("--db-path", help="Custom database path")
+    args = parser.parse_args()
+
+    asyncio.run(run_migration(args.db_path))
+
+
+if __name__ == "__main__":
+    main()

crawl4ai/model_loader.py

@@ -2,164 +2,157 @@ from functools import lru_cache
 from pathlib import Path
 import subprocess, os
 import shutil
-import tarfile
-from crawl4ai.config import MODEL_REPO_BRANCH
+from .model_loader import *
 import argparse
-import urllib.request
+from crawl4ai.config import MODEL_REPO_BRANCH
+
 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
 
+
 @lru_cache()
 def get_available_memory(device):
     import torch
-    if device.type == 'cuda':
+
+    if device.type == "cuda":
         return torch.cuda.get_device_properties(device).total_memory
-    elif device.type == 'mps':      
-        return 48 * 1024 ** 3  # Assuming 8GB for MPS, as a conservative estimate
+    elif device.type == "mps":
+        return 48 * 1024**3  # Assuming 8GB for MPS, as a conservative estimate
     else:
         return 0
 
+
 @lru_cache()
 def calculate_batch_size(device):
     available_memory = get_available_memory(device)
-    
-    if device.type == 'cpu':
+
+    if device.type == "cpu":
         return 16
-    elif device.type in ['cuda', 'mps']:
+    elif device.type in ["cuda", "mps"]:
         # Adjust these thresholds based on your model size and available memory
-        if available_memory >= 31 * 1024 ** 3:  # > 32GB
+        if available_memory >= 31 * 1024**3:  # > 32GB
             return 256
-        elif available_memory >= 15 * 1024 ** 3:  # > 16GB to 32GB
+        elif available_memory >= 15 * 1024**3:  # > 16GB to 32GB
             return 128
-        elif available_memory >= 8 * 1024 ** 3:  # 8GB to 16GB
+        elif available_memory >= 8 * 1024**3:  # 8GB to 16GB
             return 64
         else:
             return 32
     else:
-        return 16  # Default batch size   
-    
+        return 16  # Default batch size
+
+
 @lru_cache()
 def get_device():
     import torch
-    if torch.cuda.is_available():
-        device = torch.device('cuda')
-    elif torch.backends.mps.is_available():
-        device = torch.device('mps')
-    else:
-        device = torch.device('cpu')
-    return device   
-    
-def set_model_device(model):
-    device = get_device()
-    model.to(device)    
-    return model, device
 
-@lru_cache()
-def get_home_folder():
-    home_folder = os.path.join(Path.home(), ".crawl4ai")
-    os.makedirs(home_folder, exist_ok=True)
-    os.makedirs(f"{home_folder}/cache", exist_ok=True)
-    os.makedirs(f"{home_folder}/models", exist_ok=True)
-    return home_folder 
-
-@lru_cache()
-def load_bert_base_uncased():
-    from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
-    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', resume_download=None)
-    model = BertModel.from_pretrained('bert-base-uncased', resume_download=None)
-    model.eval()
-    model, device = set_model_device(model)
-    return tokenizer, model
-
-@lru_cache()
-def load_bge_small_en_v1_5():
-    from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
-    tokenizer = AutoTokenizer.from_pretrained('BAAI/bge-small-en-v1.5', resume_download=None)
-    model = AutoModel.from_pretrained('BAAI/bge-small-en-v1.5', resume_download=None)
-    model.eval()
-    model, device = set_model_device(model)
-    return tokenizer, model
-
-@lru_cache()
-def load_onnx_all_MiniLM_l6_v2():
-    from crawl4ai.onnx_embedding import DefaultEmbeddingModel
-
-    model_path = "models/onnx.tar.gz"
-    model_url = "https://unclecode-files.s3.us-west-2.amazonaws.com/onnx.tar.gz"
-    __location__ = os.path.realpath(
-        os.path.join(os.getcwd(), os.path.dirname(__file__)))
-    download_path = os.path.join(__location__, model_path)
-    onnx_dir = os.path.join(__location__, "models/onnx")
-    
-    # Create the models directory if it does not exist
-    os.makedirs(os.path.dirname(download_path), exist_ok=True)
-
-    # Download the tar.gz file if it does not exist
-    if not os.path.exists(download_path):
-        def download_with_progress(url, filename):
-            def reporthook(block_num, block_size, total_size):
-                downloaded = block_num * block_size
-                percentage = 100 * downloaded / total_size
-                if downloaded < total_size:
-                    print(f"\rDownloading: {percentage:.2f}% ({downloaded / (1024 * 1024):.2f} MB of {total_size / (1024 * 1024):.2f} MB)", end='')
-                else:
-                    print("\rDownload complete!")
-
-            urllib.request.urlretrieve(url, filename, reporthook)
-
-        download_with_progress(model_url, download_path)
-
-    # Extract the tar.gz file if the onnx directory does not exist
-    if not os.path.exists(onnx_dir):
-        with tarfile.open(download_path, "r:gz") as tar:
-            tar.extractall(path=os.path.join(__location__, "models"))
-        
-        # remove the tar.gz file
-        os.remove(download_path)
-    
-    
-    
-    model = DefaultEmbeddingModel()
-    return model
-
-@lru_cache()
-def load_text_classifier():
-    from transformers import AutoTokenizer, AutoModelForSequenceClassification
-    from transformers import pipeline
-    import torch
-
-    tokenizer = AutoTokenizer.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
-    model = AutoModelForSequenceClassification.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
-    model.eval()
-    model, device = set_model_device(model)
-    pipe = pipeline("text-classification", model=model, tokenizer=tokenizer)
-    return pipe
-
-@lru_cache()
-def load_text_multilabel_classifier():
-    from transformers import AutoModelForSequenceClassification, AutoTokenizer
-    import numpy as np
-    from scipy.special import expit
-    import torch
-
-    # Check for available device: CUDA, MPS (for Apple Silicon), or CPU
     if torch.cuda.is_available():
         device = torch.device("cuda")
     elif torch.backends.mps.is_available():
         device = torch.device("mps")
     else:
-        return load_spacy_model(), torch.device("cpu")
+        device = torch.device("cpu")
+    return device
 
 
+def set_model_device(model):
+    device = get_device()
+    model.to(device)
+    return model, device
+
+
+@lru_cache()
+def get_home_folder():
+    home_folder = os.path.join(
+        os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
+    )
+    os.makedirs(home_folder, exist_ok=True)
+    os.makedirs(f"{home_folder}/cache", exist_ok=True)
+    os.makedirs(f"{home_folder}/models", exist_ok=True)
+    return home_folder
+
+
+@lru_cache()
+def load_bert_base_uncased():
+    from transformers import BertTokenizer, BertModel
+
+    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", resume_download=None)
+    model = BertModel.from_pretrained("bert-base-uncased", resume_download=None)
+    model.eval()
+    model, device = set_model_device(model)
+    return tokenizer, model
+
+
+@lru_cache()
+def load_HF_embedding_model(model_name="BAAI/bge-small-en-v1.5") -> tuple:
+    """Load the Hugging Face model for embedding.
+
+    Args:
+        model_name (str, optional): The model name to load. Defaults to "BAAI/bge-small-en-v1.5".
+
+    Returns:
+        tuple: The tokenizer and model.
+    """
+    from transformers import AutoTokenizer, AutoModel
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name, resume_download=None)
+    model = AutoModel.from_pretrained(model_name, resume_download=None)
+    model.eval()
+    model, device = set_model_device(model)
+    return tokenizer, model
+
+
+@lru_cache()
+def load_text_classifier():
+    from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    from transformers import pipeline
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        "dstefa/roberta-base_topic_classification_nyt_news"
+    )
+    model = AutoModelForSequenceClassification.from_pretrained(
+        "dstefa/roberta-base_topic_classification_nyt_news"
+    )
+    model.eval()
+    model, device = set_model_device(model)
+    pipe = pipeline("text-classification", model=model, tokenizer=tokenizer)
+    return pipe
+
+
+@lru_cache()
+def load_text_multilabel_classifier():
+    from transformers import AutoModelForSequenceClassification, AutoTokenizer
+    from scipy.special import expit
+    import torch
+
+    # # Check for available device: CUDA, MPS (for Apple Silicon), or CPU
+    # if torch.cuda.is_available():
+    #     device = torch.device("cuda")
+    # elif torch.backends.mps.is_available():
+    #     device = torch.device("mps")
+    # else:
+    #     device = torch.device("cpu")
+    #     # return load_spacy_model(), torch.device("cpu")
+
     MODEL = "cardiffnlp/tweet-topic-21-multi"
     tokenizer = AutoTokenizer.from_pretrained(MODEL, resume_download=None)
-    model = AutoModelForSequenceClassification.from_pretrained(MODEL, resume_download=None)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        MODEL, resume_download=None
+    )
     model.eval()
     model, device = set_model_device(model)
     class_mapping = model.config.id2label
 
     def _classifier(texts, threshold=0.5, max_length=64):
-        tokens = tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=max_length)
-        tokens = {key: val.to(device) for key, val in tokens.items()}  # Move tokens to the selected device
+        tokens = tokenizer(
+            texts,
+            return_tensors="pt",
+            padding=True,
+            truncation=True,
+            max_length=max_length,
+        )
+        tokens = {
+            key: val.to(device) for key, val in tokens.items()
+        }  # Move tokens to the selected device
 
         with torch.no_grad():
             output = model(**tokens)
@@ -170,73 +163,91 @@ def load_text_multilabel_classifier():
 
         batch_labels = []
         for prediction in predictions:
-            labels = [class_mapping[i] for i, value in enumerate(prediction) if value == 1]
+            labels = [
+                class_mapping[i] for i, value in enumerate(prediction) if value == 1
+            ]
             batch_labels.append(labels)
 
         return batch_labels
 
     return _classifier, device
 
+
 @lru_cache()
 def load_nltk_punkt():
     import nltk
+
     try:
-        nltk.data.find('tokenizers/punkt')
+        nltk.data.find("tokenizers/punkt")
     except LookupError:
-        nltk.download('punkt')
-    return nltk.data.find('tokenizers/punkt')
+        nltk.download("punkt")
+    return nltk.data.find("tokenizers/punkt")
 
 
 @lru_cache()
 def load_spacy_model():
     import spacy
+
     name = "models/reuters"
     home_folder = get_home_folder()
-    model_folder = os.path.join(home_folder, name)
-    
-    # Check if the model directory already exists
-    if not (Path(model_folder).exists() and any(Path(model_folder).iterdir())):
-        repo_url = "https://github.com/unclecode/crawl4ai.git"
-        # branch = "main"
-        branch = MODEL_REPO_BRANCH 
-        repo_folder = os.path.join(home_folder, "crawl4ai")
-        model_folder = os.path.join(home_folder, name)
+    model_folder = Path(home_folder) / name
 
-        # print("[LOG] ⏬ Downloading Spacy model for the first time...")
+    # Check if the model directory already exists
+    if not (model_folder.exists() and any(model_folder.iterdir())):
+        repo_url = "https://github.com/unclecode/crawl4ai.git"
+        branch = MODEL_REPO_BRANCH
+        repo_folder = Path(home_folder) / "crawl4ai"
+
+        print("[LOG] ⏬ Downloading Spacy model for the first time...")
 
         # Remove existing repo folder if it exists
-        if Path(repo_folder).exists():
-            shutil.rmtree(repo_folder)
-            shutil.rmtree(model_folder)
+        if repo_folder.exists():
+            try:
+                shutil.rmtree(repo_folder)
+                if model_folder.exists():
+                    shutil.rmtree(model_folder)
+            except PermissionError:
+                print(
+                    "[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:"
+                )
+                print(f"- {repo_folder}")
+                print(f"- {model_folder}")
+                return None
 
         try:
             # Clone the repository
             subprocess.run(
-                ["git", "clone", "-b", branch, repo_url, repo_folder],
+                ["git", "clone", "-b", branch, repo_url, str(repo_folder)],
                 stdout=subprocess.DEVNULL,
                 stderr=subprocess.DEVNULL,
-                check=True
+                check=True,
             )
 
             # Create the models directory if it doesn't exist
-            models_folder = os.path.join(home_folder, "models")
-            os.makedirs(models_folder, exist_ok=True)
+            models_folder = Path(home_folder) / "models"
+            models_folder.mkdir(parents=True, exist_ok=True)
 
             # Copy the reuters model folder to the models directory
-            source_folder = os.path.join(repo_folder, "models/reuters")
+            source_folder = repo_folder / "models" / "reuters"
             shutil.copytree(source_folder, model_folder)
 
             # Remove the cloned repository
             shutil.rmtree(repo_folder)
 
-            # Print completion message
-            # print("[LOG] ✅ Spacy Model downloaded successfully")
+            print("[LOG] ✅ Spacy Model downloaded successfully")
         except subprocess.CalledProcessError as e:
             print(f"An error occurred while cloning the repository: {e}")
+            return None
         except Exception as e:
             print(f"An error occurred: {e}")
+            return None
+
+    try:
+        return spacy.load(str(model_folder))
+    except Exception as e:
+        print(f"Error loading spacy model: {e}")
+        return None
 
-    return spacy.load(model_folder)
 
 def download_all_models(remove_existing=False):
     """Download all models required for Crawl4AI."""
@@ -266,14 +277,20 @@ def download_all_models(remove_existing=False):
     load_nltk_punkt()
     print("[LOG] ✅ All models downloaded successfully.")
 
+
 def main():
     print("[LOG] Welcome to the Crawl4AI Model Downloader!")
     print("[LOG] This script will download all the models required for Crawl4AI.")
     parser = argparse.ArgumentParser(description="Crawl4AI Model Downloader")
-    parser.add_argument('--remove-existing', action='store_true', help="Remove existing models before downloading")
+    parser.add_argument(
+        "--remove-existing",
+        action="store_true",
+        help="Remove existing models before downloading",
+    )
     args = parser.parse_args()
-    
+
     download_all_models(remove_existing=args.remove_existing)
 
+
 if __name__ == "__main__":
     main()

crawl4ai/models.py

@@ -1,10 +1,129 @@
-from pydantic import BaseModel, HttpUrl
-from typing import List, Dict, Optional
+from pydantic import BaseModel, HttpUrl, PrivateAttr, ConfigDict
+from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
+from typing import AsyncGenerator
+from typing import Generic, TypeVar
+from enum import Enum
+from dataclasses import dataclass
+from .ssl_certificate import SSLCertificate
+from datetime import datetime
+from datetime import timedelta
+
+
+###############################
+# Dispatcher Models
+###############################
+@dataclass
+class DomainState:
+    last_request_time: float = 0
+    current_delay: float = 0
+    fail_count: int = 0
+
+
+@dataclass
+class CrawlerTaskResult:
+    task_id: str
+    url: str
+    result: "CrawlResult"
+    memory_usage: float
+    peak_memory: float
+    start_time: Union[datetime, float]
+    end_time: Union[datetime, float]
+    error_message: str = ""
+    retry_count: int = 0
+    wait_time: float = 0.0
+    
+    @property
+    def success(self) -> bool:
+        return self.result.success
+
+class CrawlStatus(Enum):
+    QUEUED = "QUEUED"
+    IN_PROGRESS = "IN_PROGRESS"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+
+@dataclass
+class CrawlStats:
+    task_id: str
+    url: str
+    status: CrawlStatus
+    start_time: Optional[Union[datetime, float]] = None
+    end_time: Optional[Union[datetime, float]] = None
+    memory_usage: float = 0.0
+    peak_memory: float = 0.0
+    error_message: str = ""
+    wait_time: float = 0.0
+    retry_count: int = 0
+    counted_requeue: bool = False
+
+    @property
+    def duration(self) -> str:
+        if not self.start_time:
+            return "0:00"
+            
+        # Convert start_time to datetime if it's a float
+        start = self.start_time
+        if isinstance(start, float):
+            start = datetime.fromtimestamp(start)
+            
+        # Get end time or use current time
+        end = self.end_time or datetime.now()
+        # Convert end_time to datetime if it's a float
+        if isinstance(end, float):
+            end = datetime.fromtimestamp(end)
+            
+        duration = end - start
+        return str(timedelta(seconds=int(duration.total_seconds())))
+
+class DisplayMode(Enum):
+    DETAILED = "DETAILED"
+    AGGREGATED = "AGGREGATED"
+
+
+###############################
+# Crawler Models
+###############################
+@dataclass
+class TokenUsage:
+    completion_tokens: int = 0
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+    completion_tokens_details: Optional[dict] = None
+    prompt_tokens_details: Optional[dict] = None
 
 class UrlModel(BaseModel):
     url: HttpUrl
     forced: bool = False
 
+class MarkdownGenerationResult(BaseModel):
+    raw_markdown: str
+    markdown_with_citations: str
+    references_markdown: str
+    fit_markdown: Optional[str] = None
+    fit_html: Optional[str] = None
+
+    def __str__(self):
+        return self.raw_markdown
+
+@dataclass
+class TraversalStats:
+    """Statistics for the traversal process"""
+
+    start_time: datetime = datetime.now()
+    urls_processed: int = 0
+    urls_failed: int = 0
+    urls_skipped: int = 0
+    total_depth_reached: int = 0
+    current_depth: int = 0
+
+class DispatchResult(BaseModel):
+    task_id: str
+    memory_usage: float
+    peak_memory: float
+    start_time: Union[datetime, float]
+    end_time: Union[datetime, float]
+    error_message: str = ""
+
 class CrawlResult(BaseModel):
     url: str
     html: str
@@ -12,8 +131,233 @@ class CrawlResult(BaseModel):
     cleaned_html: Optional[str] = None
     media: Dict[str, List[Dict]] = {}
     links: Dict[str, List[Dict]] = {}
+    downloaded_files: Optional[List[str]] = None
+    js_execution_result: Optional[Dict[str, Any]] = None
     screenshot: Optional[str] = None
-    markdown: Optional[str] = None
+    pdf: Optional[bytes] = None
+    _markdown: Optional[MarkdownGenerationResult] = PrivateAttr(default=None)
     extracted_content: Optional[str] = None
     metadata: Optional[dict] = None
-    error_message: Optional[str] = None
+    error_message: Optional[str] = None
+    session_id: Optional[str] = None
+    response_headers: Optional[dict] = None
+    status_code: Optional[int] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+    dispatch_result: Optional[DispatchResult] = None
+    redirected_url: Optional[str] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    # class Config:
+    #     arbitrary_types_allowed = True
+
+# NOTE: The StringCompatibleMarkdown class, custom __init__ method, property getters/setters,
+# and model_dump override all exist to support a smooth transition from markdown as a string
+# to markdown as a MarkdownGenerationResult object, while maintaining backward compatibility.
+# 
+# This allows code that expects markdown to be a string to continue working, while also
+# providing access to the full MarkdownGenerationResult object's properties.
+# 
+# The markdown_v2 property is deprecated and raises an error directing users to use markdown.
+# 
+# When backward compatibility is no longer needed in future versions, this entire mechanism
+# can be simplified to a standard field with no custom accessors or serialization logic.
+    
+    def __init__(self, **data):
+        markdown_result = data.pop('markdown', None)
+        super().__init__(**data)
+        if markdown_result is not None:
+            self._markdown = (
+                MarkdownGenerationResult(**markdown_result)
+                if isinstance(markdown_result, dict)
+                else markdown_result
+            )
+    
+    @property
+    def markdown(self):
+        """
+        Property that returns a StringCompatibleMarkdown object that behaves like
+        a string but also provides access to MarkdownGenerationResult attributes.
+        
+        This approach allows backward compatibility with code that expects 'markdown'
+        to be a string, while providing access to the full MarkdownGenerationResult.
+        """
+        if self._markdown is None:
+            return None
+        return StringCompatibleMarkdown(self._markdown)
+    
+    @markdown.setter
+    def markdown(self, value):
+        """
+        Setter for the markdown property.
+        """
+        self._markdown = value
+    
+    @property
+    def markdown_v2(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+
+        This property exists to inform users that 'markdown_v2' has been
+        deprecated and they should use 'markdown' instead.
+        """
+        raise AttributeError(
+            "The 'markdown_v2' attribute is deprecated and has been removed. "
+            """Please use 'markdown' instead, which now returns a MarkdownGenerationResult, with
+            following properties:
+            - raw_markdown: The raw markdown string
+            - markdown_with_citations: The markdown string with citations
+            - references_markdown: The markdown string with references
+            - fit_markdown: The markdown string with fit text
+            """
+        )
+    
+    @property
+    def fit_markdown(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+        """
+        raise AttributeError(
+            "The 'fit_markdown' attribute is deprecated and has been removed. "
+            "Please use 'markdown.fit_markdown' instead."
+        )
+    
+    @property
+    def fit_html(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+        """
+        raise AttributeError(
+            "The 'fit_html' attribute is deprecated and has been removed. "
+            "Please use 'markdown.fit_html' instead."
+        )
+
+    def model_dump(self, *args, **kwargs):
+        """
+        Override model_dump to include the _markdown private attribute in serialization.
+        
+        This override is necessary because:
+        1. PrivateAttr fields are excluded from serialization by default
+        2. We need to maintain backward compatibility by including the 'markdown' field
+           in the serialized output
+        3. We're transitioning from 'markdown_v2' to enhancing 'markdown' to hold
+           the same type of data
+        
+        Future developers: This method ensures that the markdown content is properly
+        serialized despite being stored in a private attribute. If the serialization
+        requirements change, this is where you would update the logic.
+        """
+        result = super().model_dump(*args, **kwargs)
+        if self._markdown is not None:
+            result["markdown"] = self._markdown.model_dump() 
+        return result
+
+class StringCompatibleMarkdown(str):
+    """A string subclass that also provides access to MarkdownGenerationResult attributes"""
+    def __new__(cls, markdown_result):
+        return super().__new__(cls, markdown_result.raw_markdown)
+    
+    def __init__(self, markdown_result):
+        self._markdown_result = markdown_result
+    
+    def __getattr__(self, name):
+        return getattr(self._markdown_result, name)
+
+CrawlResultT = TypeVar('CrawlResultT', bound=CrawlResult)
+
+class CrawlResultContainer(Generic[CrawlResultT]):
+    def __init__(self, results: Union[CrawlResultT, List[CrawlResultT]]):
+        # Normalize to a list
+        if isinstance(results, list):
+            self._results = results
+        else:
+            self._results = [results]
+
+    def __iter__(self):
+        return iter(self._results)
+
+    def __getitem__(self, index):
+        return self._results[index]
+
+    def __len__(self):
+        return len(self._results)
+
+    def __getattr__(self, attr):
+        # Delegate attribute access to the first element.
+        if self._results:
+            return getattr(self._results[0], attr)
+        raise AttributeError(f"{self.__class__.__name__} object has no attribute '{attr}'")
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._results!r})"
+
+RunManyReturn = Union[
+    CrawlResultContainer[CrawlResultT],
+    AsyncGenerator[CrawlResultT, None]
+]
+
+
+# END of backward compatibility code for markdown/markdown_v2.
+# When removing this code in the future, make sure to:
+# 1. Replace the private attribute and property with a standard field
+# 2. Update any serialization logic that might depend on the current behavior
+
+class AsyncCrawlResponse(BaseModel):
+    html: str
+    response_headers: Dict[str, str]
+    js_execution_result: Optional[Dict[str, Any]] = None
+    status_code: int
+    screenshot: Optional[str] = None
+    pdf_data: Optional[bytes] = None
+    get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
+    downloaded_files: Optional[List[str]] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+    redirected_url: Optional[str] = None
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    # class Config:
+    #     arbitrary_types_allowed = True
+
+###############################
+# Scraping Models
+###############################
+class MediaItem(BaseModel):
+    src: Optional[str] = ""
+    data: Optional[str] = ""
+    alt: Optional[str] = ""
+    desc: Optional[str] = ""
+    score: Optional[int] = 0
+    type: str = "image"
+    group_id: Optional[int] = 0
+    format: Optional[str] = None
+    width: Optional[int] = None
+
+
+class Link(BaseModel):
+    href: Optional[str] = ""
+    text: Optional[str] = ""
+    title: Optional[str] = ""
+    base_domain: Optional[str] = ""
+
+
+class Media(BaseModel):
+    images: List[MediaItem] = []
+    videos: List[
+        MediaItem
+    ] = []  # Using MediaItem model for now, can be extended with Video model if needed
+    audios: List[
+        MediaItem
+    ] = []  # Using MediaItem model for now, can be extended with Audio model if needed
+    tables: List[Dict] = []  # Table data extracted from HTML tables
+
+
+class Links(BaseModel):
+    internal: List[Link] = []
+    external: List[Link] = []
+
+
+class ScrapingResult(BaseModel):
+    cleaned_html: str
+    success: bool
+    media: Media = Media()
+    links: Links = Links()
+    metadata: Dict[str, Any] = {}

crawl4ai/models/onnx/config.json

@@ -1,25 +0,0 @@
-{
-  "_name_or_path": "sentence-transformers/all-MiniLM-L6-v2",
-  "architectures": [
-    "BertModel"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
-  "gradient_checkpointing": false,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 384,
-  "initializer_range": 0.02,
-  "intermediate_size": 1536,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 6,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "transformers_version": "4.27.4",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 30522
-}

crawl4ai/models/onnx/special_tokens_map.json

@@ -1,7 +0,0 @@
-{
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
-}

crawl4ai/models/onnx/tokenizer_config.json

@@ -1,15 +0,0 @@
-{
-  "cls_token": "[CLS]",
-  "do_basic_tokenize": true,
-  "do_lower_case": true,
-  "mask_token": "[MASK]",
-  "model_max_length": 512,
-  "never_split": null,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "special_tokens_map_file": "/Users/hammad/.cache/huggingface/hub/models--sentence-transformers--all-MiniLM-L6-v2/snapshots/7dbbc90392e2f80f3d3c277d6e90027e55de9125/special_tokens_map.json",
-  "strip_accents": null,
-  "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
-  "unk_token": "[UNK]"
-}

crawl4ai/onnx_embedding.py

@@ -1,50 +0,0 @@
-# A dependency-light way to run the onnx model
-
-
-import numpy as np
-from typing import List
-import os
-
-__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
-MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2"
-
-def normalize(v):
-    norm = np.linalg.norm(v, axis=1)
-    norm[norm == 0] = 1e-12
-    return v / norm[:, np.newaxis]
-
-# Sampel implementation of the default sentence-transformers model using ONNX
-class DefaultEmbeddingModel():
-
-    def __init__(self):
-        from tokenizers import Tokenizer
-        import onnxruntime as ort
-        # max_seq_length = 256, for some reason sentence-transformers uses 256 even though the HF config has a max length of 128
-        # https://github.com/UKPLab/sentence-transformers/blob/3e1929fddef16df94f8bc6e3b10598a98f46e62d/docs/_static/html/models_en_sentence_embeddings.html#LL480
-        self.tokenizer = Tokenizer.from_file(os.path.join(__location__, "models/onnx/tokenizer.json"))
-        self.tokenizer.enable_truncation(max_length=256)
-        self.tokenizer.enable_padding(pad_id=0, pad_token="[PAD]", length=256)
-        self.model = ort.InferenceSession(os.path.join(__location__,"models/onnx/model.onnx"))
-        
-
-    def __call__(self, documents: List[str], batch_size: int = 32):
-        all_embeddings = []
-        for i in range(0, len(documents), batch_size):
-            batch = documents[i:i + batch_size]
-            encoded = [self.tokenizer.encode(d) for d in batch]
-            input_ids = np.array([e.ids for e in encoded])
-            attention_mask = np.array([e.attention_mask for e in encoded])
-            onnx_input = {
-                "input_ids": np.array(input_ids, dtype=np.int64),
-                "attention_mask": np.array(attention_mask, dtype=np.int64),
-                "token_type_ids": np.array([np.zeros(len(e), dtype=np.int64) for e in input_ids], dtype=np.int64),
-            }
-            model_output = self.model.run(None, onnx_input)
-            last_hidden_state = model_output[0]
-            # Perform mean pooling with attention weighting
-            input_mask_expanded = np.broadcast_to(np.expand_dims(attention_mask, -1), last_hidden_state.shape)
-            embeddings = np.sum(last_hidden_state * input_mask_expanded, 1) / np.clip(input_mask_expanded.sum(1), a_min=1e-9, a_max=None)
-            embeddings = normalize(embeddings).astype(np.float32)
-            all_embeddings.append(embeddings)
-        return np.concatenate(all_embeddings)
-