feat(pipeline): add high-level Crawler utility class for simplified web crawling

Add new Crawler class that provides a simplified interface for both single and batch URL crawling operations. Key features include:
- Simple single URL crawling with configurable options
- Parallel batch crawling with concurrency control
- Shared browser hub support for resource efficiency
- Progress tracking and custom retry strategies
- Comprehensive error handling and retry logic

Remove demo and extended test files in favor of new focused test suite.

.github/DISCUSSION_TEMPLATE/feature-requests.yml

@@ -0,0 +1,59 @@
+title: "[Feature Request]: "
+labels: ["⚙️ New"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for your interest in suggesting a new feature! Before you submit, please take a moment to check if already exists in
+        this discussions category to avoid duplicates. 😊
+
+  - type: textarea
+    id: needs_to_be_done
+    attributes:
+      label: What needs to be done?
+      description: Please describe the feature or functionality you'd like to see.
+      placeholder: "e.g., Return alt text along with images scraped from a webpages in Result"
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem_to_solve
+    attributes:
+      label: What problem does this solve?
+      description: Explain the pain point or issue this feature will help address.
+      placeholder: "e.g., Bypass Captchas added by cloudflare"
+    validations:
+      required: true
+
+  - type: textarea
+    id: target_users
+    attributes:
+      label: Target users/beneficiaries
+      description: Who would benefit from this feature? (e.g., specific teams, developers, users, etc.)
+      placeholder: "e.g., Marketing teams, developers"
+    validations:
+      required: false
+
+  - type: textarea
+    id: current_workarounds
+    attributes:
+      label: Current alternatives/workarounds
+      description: Are there any existing solutions or workarounds? How does this feature improve upon them?
+      placeholder: "e.g., Users manually select the css classes mapped to data fields to extract them"
+    validations:
+      required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        ### 💡 Implementation Ideas
+
+  - type: textarea
+    id: proposed_approach
+    attributes:
+      label: Proposed approach
+      description: Share any ideas you have for how this feature could be implemented. Point out any challenges your foresee
+       and the success metrics for this feature
+      placeholder: "e.g., Implement a breadth first traversal algorithm for scraper"
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,127 @@
+name: Bug Report
+description: Report a bug with the Crawl4AI.
+title: "[Bug]: "
+labels: ["🐞 Bug","🩺 Needs Triage"]
+body:
+  - type: input
+    id: crawl4ai_version
+    attributes:
+      label: crawl4ai version
+      description: Specify the version of crawl4ai you are using.
+      placeholder: "e.g., 2.0.0"
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected_behavior
+    attributes:
+      label: Expected Behavior
+      description: Describe what you expected to happen.
+      placeholder: "Provide a detailed explanation of the expected outcome."
+    validations:
+      required: true
+
+  - type: textarea
+    id: current_behavior
+    attributes:
+      label: Current Behavior
+      description: Describe what is happening instead of the expected behavior.
+      placeholder: "Describe the actual result or issue you encountered."
+    validations:
+      required: true
+
+  - type: dropdown
+    id: reproducible
+    attributes:
+      label: Is this reproducible?
+      description: Indicate whether this bug can be reproduced consistently.
+      options:
+        - "Yes"
+        - "No"
+    validations:
+      required: true
+
+  - type: textarea
+    id: inputs
+    attributes:
+      label: Inputs Causing the Bug
+      description: Provide details about the inputs causing the issue.
+      placeholder: |
+        - URL(s): 
+        - Settings used: 
+        - Input data (if applicable): 
+      render: bash
+  
+  - type: textarea
+    id: steps_to_reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: Provide step-by-step instructions to reproduce the issue.
+      placeholder: |
+        1. Go to...
+        2. Click on...
+        3. Observe the issue...
+      render: bash
+  
+  - type: textarea
+    id: code_snippets
+    attributes:
+      label: Code snippets
+      description: Provide code snippets(if any). Add comments as necessary
+      placeholder: print("Hello world")
+      render: python
+
+  # Header Section with Title
+  - type: markdown
+    attributes:
+      value: |
+          ## Supporting Information
+          Please provide the following details to help us understand and resolve your issue. This will assist us in reproducing and diagnosing the problem
+
+  - type: input
+    id: os
+    attributes:
+      label: OS
+      description: Please provide the operating system & distro where the issue occurs.
+      placeholder: "e.g., Windows, macOS, Linux"
+    validations:
+      required: true
+
+  - type: input
+    id: python_version
+    attributes:
+      label: Python version
+      description: Specify the Python version being used.
+      placeholder: "e.g., 3.8.5"
+    validations:
+      required: true
+
+  # Browser Field
+  - type: input
+    id: browser
+    attributes:
+      label: Browser
+      description: Provide the name of the browser you are using.
+      placeholder: "e.g., Chrome, Firefox, Safari"
+    validations:
+      required: false
+
+  # Browser Version Field
+  - type: input
+    id: browser_version
+    attributes:
+      label: Browser version
+      description: Provide the version of the browser you are using.
+      placeholder: "e.g., 91.0.4472.124"
+    validations:
+      required: false
+
+  # Error Logs Field (Text Area)
+  - type: textarea
+    id: error_logs
+    attributes:
+      label: Error logs & Screenshots (if applicable)
+      description: If you encountered any errors, please provide the error logs. Attach any relevant screenshots to help us understand the issue.
+      placeholder: "Paste error logs here and attach your screenshots"
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Feature Requests
+    url: https://github.com/unclecode/crawl4ai/discussions/categories/feature-requests
+    about: "Suggest new features or enhancements for Crawl4AI"
+  - name: Forums - Q&A
+    url: https://github.com/unclecode/crawl4ai/discussions/categories/forums-q-a
+    about: "Ask questions or engage in general discussions about Crawl4AI"

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+## Summary
+Please include a summary of the change and/or which issues are fixed.
+
+eg: `Fixes #123` (Tag GitHub issue numbers in this format, so it automatically links the issues with your PR)
+
+## List of files changed and why
+eg: quickstart.py - To update the example as per new changes
+
+## How Has This Been Tested?
+Please describe the tests that you ran to verify your changes.
+
+## Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] I have added/updated unit tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes

.gitignore

@@ -226,8 +226,35 @@ tree.md
 .local
 .do
 /plans
+plans/
+
+# Codeium
 .codeiumignore
 todo/
 
-# windsurf rules
-.windsurfrules
+# Continue development files
+.continue/
+.continuerc.json
+continue.lock
+continue_core.log
+contextProviders/
+continue_workspace/
+.continue-cache/
+continue_config.json
+
+# Continue temporary files
+.continue-temp/
+.continue-logs/
+.continue-downloads/
+
+# Continue VS Code specific
+.vscode-continue/
+.vscode-continue-cache/
+
+.prompts/
+
+.llm.env
+.private/
+
+CLAUDE_MONITOR.md
+CLAUDE.md

CHANGELOG.md

@@ -5,9 +5,170 @@ All notable changes to Crawl4AI will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-Okay, here's a detailed changelog in Markdown format, generated from the provided git diff and commit history. I've focused on user-facing changes, fixes, and features, and grouped them as requested:
+## Version 0.5.0.post5 (2025-03-14)
 
-## Version 0.4.3 (2025-01-21)
+### Added
+
+- *(crawler)* Add experimental parameters dictionary to CrawlerRunConfig to support beta features
+- *(tables)* Add comprehensive table detection and extraction functionality with scoring system
+- *(monitor)* Add real-time crawler monitoring system with memory management
+- *(content)* Add target_elements parameter for selective content extraction
+- *(browser)* Add standalone CDP browser launch capability
+- *(schema)* Add preprocess_html_for_schema utility for better HTML cleaning
+- *(api)* Add special handling for single URL requests in Docker API
+
+### Changed
+
+- *(filters)* Add reverse option to URLPatternFilter for inverting filter logic
+- *(browser)* Make CSP nonce headers optional via experimental config
+- *(browser)* Remove default cookie injection from page initialization
+- *(crawler)* Optimize response handling for single-URL processing
+- *(api)* Refactor crawl request handling to streamline processing
+- *(config)* Update default provider to gpt-4o
+- *(cache)* Change default cache_mode from aggressive to bypass in examples
+
+### Fixed
+
+- *(browser)* Clean up browser context creation code
+- *(api)* Improve code formatting in API handler
+
+### Breaking Changes
+
+- WebScrapingStrategy no longer returns 'scraped_html' in its output dictionary
+- Table extraction logic has been modified to better handle thead/tbody structures
+- Default cookie injection has been removed from page initialization
+
+## Version 0.5.0 (2025-03-02)
+
+### Added
+
+- *(profiles)* Add BrowserProfiler class for dedicated browser profile management
+- *(cli)* Add interactive profile management to CLI with rich UI
+- *(profiles)* Add ability to crawl directly from profile management interface
+- *(browser)* Support identity-based browsing with persistent profiles
+- *(deep-crawling)* Add max_pages parameter to limit the number of pages crawled in all deep crawling strategies
+- *(deep-crawling)* Add score_threshold parameter to BFS and DFS strategies to filter URLs by score
+
+### Changed
+
+- *(browser)* Refactor profile management from ManagedBrowser to BrowserProfiler class
+- *(cli)* Enhance CLI with profile selection and status display for crawling
+- *(examples)* Update identity-based browsing example to use BrowserProfiler class
+- *(docs)* Update identity-based crawling documentation
+- *(docs)* Update deep crawling documentation with max_pages and score_threshold parameters
+- *(examples)* Add example demonstrating the use of max_pages and score_threshold parameters
+
+### Fixed
+
+- *(browser)* Fix profile detection and management on different platforms
+- *(cli)* Fix CLI command structure for better user experience
+- *(deep-crawling)* Improve BFS and DFS strategies to handle page count limits more efficiently
+
+
+## Version 0.5.0 (2025-02-21)
+
+### Added
+
+- *(crawler)* [**breaking**] Add memory-adaptive dispatcher with rate limiting
+- *(scraping)* [**breaking**] Add LXML-based scraping mode for improved performance
+- *(content-filter)* Add LLMContentFilter for intelligent markdown generation
+- *(dispatcher)* [**breaking**] Add streaming support for URL processing
+- *(browser)* [**breaking**] Improve browser context management and add shared data support
+- *(config)* [**breaking**] Add streaming support and config cloning
+- *(crawler)* Add URL redirection tracking
+- *(extraction)* Add LLM-powered schema generation utility
+- *(proxy)* Add proxy configuration support to CrawlerRunConfig
+- *(robots)* Add robots.txt compliance support
+- *(release)* [**breaking**] Prepare v0.4.3 beta release
+- *(proxy)* Add proxy rotation support and documentation
+- *(browser)* Add CDP URL configuration support
+- *(demo)* Uncomment feature demos and add fake-useragent dependency
+- *(pdf)* Add PDF processing capabilities
+- *(crawler)* [**breaking**] Enhance JavaScript execution and PDF processing
+- *(docker)* Add Docker deployment configuration and API server
+- *(docker)* Add Docker service integration and config serialization
+- *(docker)* [**breaking**] Enhance Docker deployment setup and configuration
+- *(api)* Improve cache handling and add API tests
+- *(crawler)* [**breaking**] Add deep crawling capabilities with BFS strategy
+- *(proxy)* [**breaking**] Add proxy rotation strategy
+- *(deep-crawling)* Add DFS strategy and update exports; refactor CLI entry point
+- *(cli)* Add command line interface with comprehensive features
+- *(config)* Enhance serialization and add deep crawling exports
+- *(crawler)* Add HTTP crawler strategy for lightweight web scraping
+- *(docker)* [**breaking**] Implement supervisor and secure API endpoints
+- *(docker)* [**breaking**] Add JWT authentication and improve server architecture
+
+### Changed
+
+- *(browser)* Update browser channel default to 'chromium' in BrowserConfig.from_args method
+- *(crawler)* Optimize response handling and default settings
+- *(crawler)* - Update hello_world example with proper content filtering
+- - Update hello_world.py example
+- *(docs)* [**breaking**] Reorganize documentation structure and update styles
+- *(dispatcher)* [**breaking**] Migrate to modular dispatcher system with enhanced monitoring
+- *(scraping)* [**breaking**] Replace ScrapingMode enum with strategy pattern
+- *(browser)* Improve browser path management
+- *(models)* Rename final_url to redirected_url for consistency
+- *(core)* [**breaking**] Improve type hints and remove unused file
+- *(docs)* Improve code formatting in features demo
+- *(user-agent)* Improve user agent generation system
+- *(core)* [**breaking**] Reorganize project structure and remove legacy code
+- *(docker)* Clean up import statements in server.py
+- *(docker)* Remove unused models and utilities for cleaner codebase
+- *(docker)* [**breaking**] Improve server architecture and configuration
+- *(deep-crawl)* [**breaking**] Reorganize deep crawling functionality into dedicated module
+- *(deep-crawling)* [**breaking**] Reorganize deep crawling strategies and add new implementations
+- *(crawling)* [**breaking**] Improve type hints and code cleanup
+- *(crawler)* [**breaking**] Improve HTML handling and cleanup codebase
+- *(crawler)* [**breaking**] Remove content filter functionality
+- *(examples)* Update API usage in features demo
+- *(config)* [**breaking**] Enhance serialization and config handling
+
+### Docs
+
+- Add Code of Conduct for the project (#410)
+
+### Documentation
+
+- *(extraction)* Add clarifying comments for CSS selector behavior
+- *(readme)* Update personal story and project vision
+- *(urls)* [**breaking**] Update documentation URLs to new domain
+- *(api)* Add streaming mode documentation and examples
+- *(readme)* Update version and feature announcements for v0.4.3b1
+- *(examples)* Update demo scripts and fix output formats
+- *(examples)* Update v0.4.3 features demo to v0.4.3b2
+- *(readme)* Update version references and fix links
+- *(multi-url)* [**breaking**] Improve documentation clarity and update examples
+- *(examples)* Update proxy rotation demo and disable other demos
+- *(api)* Improve formatting and readability of API documentation
+- *(examples)* Add SERP API project example
+- *(urls)* Update documentation URLs to new domain
+- *(readme)* Resolve merge conflict and update version info
+
+### Fixed
+
+- *(browser)* Update default browser channel to chromium and simplify channel selection logic
+- *(browser)* [**breaking**] Default to Chromium channel for new headless mode (#387)
+- *(browser)* Resolve merge conflicts in browser channel configuration
+- Prevent memory leaks by ensuring proper closure of Playwright pages
+- Not working long page screenshot (#403)
+- *(extraction)* JsonCss selector and crawler improvements
+- *(models)* [**breaking**] Make model fields optional with default values
+- *(dispatcher)* Adjust memory threshold and fix dispatcher initialization
+- *(install)* Ensure proper exit after running doctor command
+
+### Miscellaneous Tasks
+
+- *(cleanup)* Remove unused files and improve type hints
+- Add .gitattributes file
+
+## License Update
+
+Crawl4AI v0.5.0 updates the license to Apache 2.0 *with a required attribution clause*.  This means you are free to use, modify, and distribute Crawl4AI (even commercially), but you *must* clearly attribute the project in any public use or distribution.  See the updated `LICENSE` file for the full legal text and specific requirements.
+
+---
+
+## Version 0.4.3b2 (2025-01-21)
 
 This release introduces several powerful new features, including robots.txt compliance, dynamic proxy support, LLM-powered schema generation, and improved documentation.
 
@@ -135,9 +296,11 @@ This release introduces several powerful new features, including robots.txt comp
 - **Multiple Element Selection**: Modified `_get_elements` in `JsonCssExtractionStrategy` to return all matching elements instead of just the first one, ensuring comprehensive extraction. ([#extraction_strategy.py](crawl4ai/extraction_strategy.py))
 - **Error Handling in Scrolling**: Added robust error handling to ensure scrolling proceeds safely even if a configuration is missing. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
 
-#### Other
-- **Git Ignore Update**: Added `/plans` to `.gitignore` for better development environment consistency. ([#.gitignore](.gitignore))
+## [0.4.267] - 2025 - 01 - 06
 
+### Added
+- **Windows Event Loop Configuration**: Introduced a utility function `configure_windows_event_loop` to resolve `NotImplementedError` for asyncio subprocesses on Windows. ([#utils.py](crawl4ai/utils.py), [#tutorials/async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md))
+- **`page_need_scroll` Method**: Added a method to determine if a page requires scrolling before taking actions in `AsyncPlaywrightCrawlerStrategy`. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
 
 ## [0.4.24] - 2024-12-31
 
@@ -281,12 +444,6 @@ This release introduces several powerful new features, including robots.txt comp
 - Fixed potential viewport mismatches by ensuring consistent use of `self.viewport_width` and `self.viewport_height` throughout the code.
 - Improved robustness of dynamic content loading to avoid timeouts and failed evaluations.
 
-
-
-
-
-
-
 ## [0.3.75] December 1, 2024
 
 ### PruningContentFilter

CODE_OF_CONDUCT.md

@@ -0,0 +1,131 @@
+# Crawl4AI Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+unclecode@crawl4ai.com. All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTORS.md

@@ -24,6 +24,14 @@ We would like to thank the following people for their contributions to Crawl4AI:
 - [NanmiCoder](https://github.com/NanmiCoder) - fix: crawler strategy exception handling and fixes [#271](https://github.com/unclecode/crawl4ai/pull/271)
 - [paulokuong](https://github.com/paulokuong) - fix: RAWL4_AI_BASE_DIRECTORY should be Path object instead of string [#298](https://github.com/unclecode/crawl4ai/pull/298)
 
+#### Feb-Alpha-1
+- [sufianuddin](https://github.com/sufianuddin) - fix: [Documentation for JsonCssExtractionStrategy](https://github.com/unclecode/crawl4ai/issues/651)
+- [tautikAg](https://github.com/tautikAg) - fix: [Markdown output has incorect spacing](https://github.com/unclecode/crawl4ai/issues/599)
+- [cardit1](https://github.com/cardit1) - fix: ['AsyncPlaywrightCrawlerStrategy' object has no attribute 'downloads_path'](https://github.com/unclecode/crawl4ai/issues/585)
+- [dmurat](https://github.com/dmurat) - fix: [ Incorrect rendering of inline code inside of links ](https://github.com/unclecode/crawl4ai/issues/583)
+- [Sparshsing](https://github.com/Sparshsing) - fix: [Relative Urls in the webpage not extracted properly ](https://github.com/unclecode/crawl4ai/issues/570)
+
+
 
 ## Other Contributors
 
@@ -31,6 +39,11 @@ We would like to thank the following people for their contributions to Crawl4AI:
 - [Shiv Kumar](https://github.com/shivkumar0757)
 - [QIN2DIM](https://github.com/QIN2DIM)
 
+#### Typo fixes
+- [ssoydan](https://github.com/ssoydan)
+- [Darshan](https://github.com/Darshan2104)
+- [tuhinmallick](https://github.com/tuhinmallick)
+
 ## Acknowledgements
 
 We also want to thank all the users who have reported bugs, suggested features, or helped in any other way to make Crawl4AI better.

Dockerfile

@@ -1,32 +1,31 @@
-# syntax=docker/dockerfile:1.4
+FROM python:3.10-slim
 
-ARG TARGETPLATFORM
-ARG BUILDPLATFORM
+# Set build arguments
+ARG APP_HOME=/app
+ARG GITHUB_REPO=https://github.com/unclecode/crawl4ai.git
+ARG GITHUB_BRANCH=main
+ARG USE_LOCAL=true
 
-# Other build arguments
-ARG PYTHON_VERSION=3.10
-
-# Base stage with system dependencies
-FROM python:${PYTHON_VERSION}-slim as base
-
-# Declare ARG variables again within the build stage
-ARG INSTALL_TYPE=all
-ARG ENABLE_GPU=false
-
-# Platform-specific labels
-LABEL maintainer="unclecode"
-LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
-LABEL version="1.0"
-
-# Environment setup
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
+ENV PYTHONFAULTHANDLER=1 \
+    PYTHONHASHSEED=random \
+    PYTHONUNBUFFERED=1 \
     PIP_NO_CACHE_DIR=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_DEFAULT_TIMEOUT=100 \
-    DEBIAN_FRONTEND=noninteractive
+    DEBIAN_FRONTEND=noninteractive \
+    REDIS_HOST=localhost \
+    REDIS_PORT=6379
+
+ARG PYTHON_VERSION=3.10
+ARG INSTALL_TYPE=default
+ARG ENABLE_GPU=false
+ARG TARGETARCH
+
+LABEL maintainer="unclecode"
+LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
+LABEL version="1.0"    
 
-# Install system dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     curl \
@@ -37,10 +36,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     pkg-config \
     python3-dev \
     libjpeg-dev \
-    libpng-dev \
+    redis-server \
+    supervisor \
     && rm -rf /var/lib/apt/lists/*
 
-# Playwright system dependencies for Linux
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libglib2.0-0 \
     libnss3 \
@@ -65,8 +64,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libatspi2.0-0 \
     && rm -rf /var/lib/apt/lists/*
 
-# GPU support if enabled and architecture is supported
-RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETPLATFORM" = "linux/amd64" ] ; then \
+RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \
     apt-get update && apt-get install -y --no-install-recommends \
     nvidia-cuda-toolkit \
     && rm -rf /var/lib/apt/lists/* ; \
@@ -74,19 +72,42 @@ else \
     echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
 fi
 
-# Create and set working directory
-WORKDIR /app
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+    echo "🦾 Installing ARM-specific optimizations"; \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libopenblas-dev \
+    && rm -rf /var/lib/apt/lists/*; \
+elif [ "$TARGETARCH" = "amd64" ]; then \
+    echo "🖥️ Installing AMD64-specific optimizations"; \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libomp-dev \
+    && rm -rf /var/lib/apt/lists/*; \
+else \
+    echo "Skipping platform-specific optimizations (unsupported platform)"; \
+fi
 
-# Copy the entire project
-COPY . .
+WORKDIR ${APP_HOME}
 
-# Install base requirements
+RUN echo '#!/bin/bash\n\
+if [ "$USE_LOCAL" = "true" ]; then\n\
+    echo "📦 Installing from local source..."\n\
+    pip install --no-cache-dir /tmp/project/\n\
+else\n\
+    echo "🌐 Installing from GitHub..."\n\
+    for i in {1..3}; do \n\
+        git clone --branch ${GITHUB_BRANCH} ${GITHUB_REPO} /tmp/crawl4ai && break || \n\
+        { echo "Attempt $i/3 failed! Taking a short break... ☕"; sleep 5; }; \n\
+    done\n\
+    pip install --no-cache-dir /tmp/crawl4ai\n\
+fi' > /tmp/install.sh && chmod +x /tmp/install.sh
+
+COPY . /tmp/project/
+
+COPY deploy/docker/supervisord.conf .
+
+COPY deploy/docker/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Install required library for FastAPI
-RUN pip install fastapi uvicorn psutil
-
-# Install ML dependencies first for better layer caching
 RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
         pip install --no-cache-dir \
             torch \
@@ -99,38 +120,37 @@ RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
         python -m nltk.downloader punkt stopwords ; \
     fi
 
-# Install the package
 RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
-        pip install ".[all]" && \
+        pip install "/tmp/project/[all]" && \
         python -m crawl4ai.model_loader ; \
     elif [ "$INSTALL_TYPE" = "torch" ] ; then \
-        pip install ".[torch]" ; \
+        pip install "/tmp/project/[torch]" ; \
     elif [ "$INSTALL_TYPE" = "transformer" ] ; then \
-        pip install ".[transformer]" && \
+        pip install "/tmp/project/[transformer]" && \
         python -m crawl4ai.model_loader ; \
     else \
-        pip install "." ; \
+        pip install "/tmp/project" ; \
     fi
+    
+RUN pip install --no-cache-dir --upgrade pip && \
+    /tmp/install.sh && \
+    python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
+    python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')"
+    
+RUN playwright install --with-deps chromium
 
-    # Install MkDocs and required plugins
-RUN pip install --no-cache-dir \
-    mkdocs \
-    mkdocs-material \
-    mkdocs-terminal \
-    pymdown-extensions
+COPY deploy/docker/* ${APP_HOME}/
 
-# Build MkDocs documentation
-RUN mkdocs build
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD bash -c '\
+    MEM=$(free -m | awk "/^Mem:/{print \$2}"); \
+    if [ $MEM -lt 2048 ]; then \
+        echo "⚠️ Warning: Less than 2GB RAM available! Your container might need a memory boost! 🚀"; \
+        exit 1; \
+    fi && \
+    redis-cli ping > /dev/null && \
+    curl -f http://localhost:8000/health || exit 1'
 
-# Install Playwright and browsers
-RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
-    playwright install chromium; \
-    elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
-    playwright install chromium; \
-    fi
-
-# Expose port
-EXPOSE 8000 11235 9222 8080
-
-# Start the FastAPI server
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "11235"]
+EXPOSE 6379
+CMD ["supervisord", "-c", "supervisord.conf"]
+    

LICENSE

@@ -48,4 +48,22 @@ You may add Your own copyright statement to Your modifications and may provide a
 
 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
 
-END OF TERMS AND CONDITIONS
+END OF TERMS AND CONDITIONS
+
+---
+Attribution Requirement
+
+All distributions, publications, or public uses of this software, or derivative works based on this software, must include the following attribution:
+
+"This product includes software developed by UncleCode (https://x.com/unclecode) as part of the Crawl4AI project (https://github.com/unclecode/crawl4ai)."
+
+This attribution must be displayed in a prominent and easily accessible location, such as:
+
+-   For software distributions: In a NOTICE file, README file, or equivalent documentation.
+-   For publications (research papers, articles, blog posts): In the acknowledgments section or a footnote.
+-   For websites/web applications: In an "About" or "Credits" section.
+-   For command-line tools: In the help/usage output.
+
+This requirement ensures proper credit is given for the use of Crawl4AI and helps promote the project.
+
+---

README.md

@@ -15,14 +15,15 @@
 [![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md)
 
 </div>
 
 Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for LLMs, AI agents, and data pipelines. Open source, flexible, and built for real-time performance, Crawl4AI empowers developers with unmatched speed, precision, and deployment ease.  
 
-[✨ Check out latest update v0.4.3bx](#-recent-updates)
+[✨ Check out latest update v0.5.0](#-recent-updates)
 
-🎉 **Version 0.4.3bx is out!** This release brings exciting new features like a Memory Dispatcher System, Streaming Support, LLM-Powered Markdown Generation, Schema Generation, and Robots.txt Compliance! [Read the release notes →](https://docs.crawl4ai.com/blog)
+🎉 **Version 0.5.0 is out!** This major release introduces Deep Crawling with BFS/DFS/BestFirst strategies, Memory-Adaptive Dispatcher, Multiple Crawling Strategies (Playwright and HTTP), Docker Deployment with FastAPI, Command-Line Interface (CLI), and more! [Read the release notes →](https://docs.crawl4ai.com/blog)
 
 <details>
 <summary>🤓 <strong>My Personal Story</strong></summary>
@@ -67,7 +68,7 @@ If you encounter any browser-related issues, you can install them manually:
 python -m playwright install --with-deps chromium
 ```
 
-2. Run a simple web crawl:
+2. Run a simple web crawl with Python:
 ```python
 import asyncio
 from crawl4ai import *
@@ -83,6 +84,18 @@ if __name__ == "__main__":
     asyncio.run(main())
 ```
 
+3. Or use the new command-line interface:
+```bash
+# Basic crawl with markdown output
+crwl https://www.nbcnews.com/business -o markdown
+
+# Deep crawl with BFS strategy, max 10 pages
+crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10
+
+# Use LLM extraction with a specific question
+crwl https://www.example.com/products -q "Extract all product prices"
+```
+
 ## ✨ Features 
 
 <details>
@@ -111,6 +124,7 @@ if __name__ == "__main__":
 
 - 🖥️ **Managed Browser**: Use user-owned browsers with full control, avoiding bot detection.
 - 🔄 **Remote Browser Control**: Connect to Chrome Developer Tools Protocol for remote, large-scale data extraction.
+- 👤 **Browser Profiler**: Create and manage persistent profiles with saved authentication states, cookies, and settings.
 - 🔒 **Session Management**: Preserve browser states and reuse them for multi-step crawling.
 - 🧩 **Proxy Support**: Seamlessly connect to proxies with authentication for secure access.
 - ⚙️ **Full Browser Control**: Modify headers, cookies, user agents, and more for tailored crawling setups.
@@ -139,10 +153,11 @@ if __name__ == "__main__":
 <details>
 <summary>🚀 <strong>Deployment</strong></summary>
 
-- 🐳 **Dockerized Setup**: Optimized Docker image with API server for easy deployment.
+- 🐳 **Dockerized Setup**: Optimized Docker image with FastAPI server for easy deployment.
+- 🔑 **Secure Authentication**: Built-in JWT token authentication for API security.
 - 🔄 **API Gateway**: One-click deployment with secure token authentication for API-based workflows.
 - 🌐 **Scalable Architecture**: Designed for mass-scale production and optimized server performance.
-- ⚙️ **DigitalOcean Deployment**: Ready-to-deploy configurations for DigitalOcean and similar platforms.
+- ☁️ **Cloud Deployment**: Ready-to-deploy configurations for major cloud platforms.
 
 </details>
 
@@ -317,9 +332,8 @@ async def main():
             url="https://docs.micronaut.io/4.7.6/guide/",
             config=run_config
         )
-        print(len(result.markdown))
-        print(len(result.fit_markdown))
-        print(len(result.markdown_v2.fit_markdown))
+        print(len(result.markdown.raw_markdown))
+        print(len(result.markdown.fit_markdown))
 
 if __name__ == "__main__":
     asyncio.run(main())
@@ -406,7 +420,7 @@ if __name__ == "__main__":
 ```python
 import os
 import asyncio
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
 from crawl4ai.extraction_strategy import LLMExtractionStrategy
 from pydantic import BaseModel, Field
 
@@ -422,7 +436,7 @@ async def main():
         extraction_strategy=LLMExtractionStrategy(
             # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2
             # provider="ollama/qwen2", api_token="no-token", 
-            provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY'), 
+            llm_config = LLMConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')), 
             schema=OpenAIModelFee.schema(),
             extraction_type="schema",
             instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
@@ -446,7 +460,7 @@ if __name__ == "__main__":
 </details>
 
 <details>
-<summary>🤖 <strong>Using You own Browswer with Custome User Profile</strong></summary>
+<summary>🤖 <strong>Using You own Browser with Custom User Profile</strong></summary>
 
 ```python
 import os, sys
@@ -486,21 +500,31 @@ async def test_news_crawl():
 
 ## ✨ Recent Updates
 
--   **🚀 New Dispatcher System**: Scale to thousands of URLs with intelligent **memory monitoring**, **concurrency control**, and optional **rate limiting**. (See `MemoryAdaptiveDispatcher`, `SemaphoreDispatcher`, `RateLimiter`, `CrawlerMonitor`)
--   **⚡ Streaming Mode**: Process results **as they arrive** instead of waiting for an entire batch to complete. (Set `stream=True` in `CrawlerRunConfig`)
--   **🤖 Enhanced LLM Integration**:
-    -   **Automatic schema generation**: Create extraction rules from HTML using OpenAI or Ollama, no manual CSS/XPath needed.
-    -   **LLM-powered Markdown filtering**: Refine your markdown output with a new `LLMContentFilter` that understands content relevance.
-    -   **Ollama Support**: Use open-source or self-hosted models for private or cost-effective extraction.
--   **🏎️ Faster Scraping Option**: New `LXMLWebScrapingStrategy` offers **10-20x speedup** for large, complex pages (experimental).
--   **🤖 robots.txt Compliance**: Respect website rules with `check_robots_txt=True` and efficient local caching.
--   **🔄 Proxy Rotation**: Built-in support for dynamic proxy switching and IP verification, with support for authenticated proxies and session persistence.
--   **➡️ URL Redirection Tracking**: The `redirected_url` field now captures the final destination after any redirects.
--   **🪞 Improved Mirroring**: The `LXMLWebScrapingStrategy` now has much greater fidelity, allowing for almost pixel-perfect mirroring of websites.
--   **📈 Enhanced Monitoring**: Track memory, CPU, and individual crawler status with `CrawlerMonitor`.
--   **📝 Improved Documentation**: More examples, clearer explanations, and updated tutorials.
+### Version 0.5.0 Major Release Highlights
 
-Read the full details in our [0.4.3bx Release Notes](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
+-   **🚀 Deep Crawling System**: Explore websites beyond initial URLs with three strategies:
+    -   **BFS Strategy**: Breadth-first search explores websites level by level
+    -   **DFS Strategy**: Depth-first search explores each branch deeply before backtracking
+    -   **BestFirst Strategy**: Uses scoring functions to prioritize which URLs to crawl next
+    -   **Page Limiting**: Control the maximum number of pages to crawl with `max_pages` parameter
+    -   **Score Thresholds**: Filter URLs based on relevance scores
+-   **⚡ Memory-Adaptive Dispatcher**: Dynamically adjusts concurrency based on system memory with built-in rate limiting
+-   **🔄 Multiple Crawling Strategies**:
+    -   **AsyncPlaywrightCrawlerStrategy**: Browser-based crawling with JavaScript support (Default)
+    -   **AsyncHTTPCrawlerStrategy**: Fast, lightweight HTTP-only crawler for simple tasks
+-   **🐳 Docker Deployment**: Easy deployment with FastAPI server and streaming/non-streaming endpoints
+-   **💻 Command-Line Interface**: New `crwl` CLI provides convenient terminal access to all features with intuitive commands and configuration options
+-   **👤 Browser Profiler**: Create and manage persistent browser profiles to save authentication states, cookies, and settings for seamless crawling of protected content
+-   **🧠 Crawl4AI Coding Assistant**: AI-powered coding assistant to answer your question for Crawl4ai, and generate proper code for crawling.
+-   **🏎️ LXML Scraping Mode**: Fast HTML parsing using the `lxml` library for improved performance
+-   **🌐 Proxy Rotation**: Built-in support for proxy switching with `RoundRobinProxyStrategy`
+-   **🤖 LLM Content Filter**: Intelligent markdown generation using LLMs
+-   **📄 PDF Processing**: Extract text, images, and metadata from PDF files
+-   **🔗 URL Redirection Tracking**: Automatically follow and record HTTP redirects
+-   **🤖 LLM Schema Generation**: Easily create extraction schemas with LLM assistance
+-   **🔍 robots.txt Compliance**: Respect website crawling rules
+
+Read the full details in our [0.5.0 Release Notes](https://docs.crawl4ai.com/blog/releases/0.5.0.html) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
 
 ## Version Numbering in Crawl4AI
 
@@ -573,9 +597,83 @@ To check our development plans and upcoming features, visit our [Roadmap](https:
 
 We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information.
 
-## 📄 License 
+I'll help modify the license section with badges. For the halftone effect, here's a version with it:
 
-Crawl4AI is released under the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE).
+Here's the updated license section:
+
+## 📄 License & Attribution
+
+This project is licensed under the Apache License 2.0 with a required attribution clause. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
+
+### Attribution Requirements
+When using Crawl4AI, you must include one of the following attribution methods:
+
+#### 1. Badge Attribution (Recommended)
+Add one of these badges to your README, documentation, or website:
+
+| Theme | Badge |
+|-------|-------|
+| **Disco Theme (Animated)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Night Theme (Dark with Neon)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Dark Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Light Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+ 
+
+HTML code for adding the badges:
+```html
+<!-- Disco Theme (Animated) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Night Theme (Dark with Neon) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Dark Theme (Classic) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Light Theme (Classic) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Simple Shield Badge -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://img.shields.io/badge/Powered%20by-Crawl4AI-blue?style=flat-square" alt="Powered by Crawl4AI"/>
+</a>
+```
+
+#### 2. Text Attribution
+Add this line to your documentation:
+```
+This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
+```
+
+## 📚 Citation
+
+If you use Crawl4AI in your research or project, please cite:
+
+```bibtex
+@software{crawl4ai2024,
+  author = {UncleCode},
+  title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper},
+  year = {2024},
+  publisher = {GitHub},
+  journal = {GitHub Repository},
+  howpublished = {\url{https://github.com/unclecode/crawl4ai}},
+  commit = {Please use the commit hash you're working with}
+}
+```
+
+Text citation format:
+```
+UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software]. 
+GitHub. https://github.com/unclecode/crawl4ai
+```
 
 ## 📧 Contact 
 

cliff.toml

@@ -0,0 +1,24 @@
+[changelog]
+# Template format
+header = """
+# Changelog\n
+All notable changes to this project will be documented in this file.\n
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n
+"""
+
+# Organize commits by type
+[git]
+conventional_commits = true
+filter_unconventional = true
+commit_parsers = [
+    { message = "^feat", group = "Added"},
+    { message = "^fix", group = "Fixed"},
+    { message = "^doc", group = "Documentation"},
+    { message = "^perf", group = "Performance"},
+    { message = "^refactor", group = "Changed"},
+    { message = "^style", group = "Changed"},
+    { message = "^test", group = "Testing"},
+    { message = "^chore\\(release\\): prepare for", skip = true},
+    { message = "^chore", group = "Miscellaneous Tasks"},
+]

crawl4ai/__init__.py

@@ -1,46 +1,122 @@
 # __init__.py
+import warnings
 
 from .async_webcrawler import AsyncWebCrawler, CacheMode
-from .async_configs import BrowserConfig, CrawlerRunConfig
+from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig, LLMConfig
+
+from .pipeline.pipeline import (
+    Pipeline,
+    create_pipeline,
+)
+from .pipeline.crawler import Crawler
+
 from .content_scraping_strategy import (
     ContentScrapingStrategy,
     WebScrapingStrategy,
     LXMLWebScrapingStrategy,
 )
+from .async_logger import (
+    AsyncLoggerBase,
+    AsyncLogger,
+)
+from .proxy_strategy import (
+    ProxyRotationStrategy,
+    RoundRobinProxyStrategy,
+)
 from .extraction_strategy import (
     ExtractionStrategy,
     LLMExtractionStrategy,
     CosineStrategy,
     JsonCssExtractionStrategy,
-    JsonXPathExtractionStrategy
+    JsonXPathExtractionStrategy,
+    JsonLxmlExtractionStrategy
 )
 from .chunking_strategy import ChunkingStrategy, RegexChunking
 from .markdown_generation_strategy import DefaultMarkdownGenerator
-from .content_filter_strategy import PruningContentFilter, BM25ContentFilter, LLMContentFilter, RelevantContentFilter
-from .models import CrawlResult, MarkdownGenerationResult
+from .content_filter_strategy import (
+    PruningContentFilter,
+    BM25ContentFilter,
+    LLMContentFilter,
+    RelevantContentFilter,
+)
+from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
+from .components.crawler_monitor import CrawlerMonitor
 from .async_dispatcher import (
     MemoryAdaptiveDispatcher,
     SemaphoreDispatcher,
     RateLimiter,
-    CrawlerMonitor,
-    DisplayMode,
-    BaseDispatcher
+    BaseDispatcher,
+)
+from .docker_client import Crawl4aiDockerClient
+from .hub import CrawlerHub
+from .browser_profiler import BrowserProfiler
+from .deep_crawling import (
+    DeepCrawlStrategy,
+    BFSDeepCrawlStrategy,
+    FilterChain,
+    URLPatternFilter,
+    DomainFilter,
+    ContentTypeFilter,
+    URLFilter,
+    FilterStats,
+    SEOFilter,
+    KeywordRelevanceScorer,
+    URLScorer,
+    CompositeScorer,
+    DomainAuthorityScorer,
+    FreshnessScorer,
+    PathDepthScorer,
+    BestFirstCrawlingStrategy,
+    DFSDeepCrawlStrategy,
+    DeepCrawlDecorator,
 )
 
+from .async_crawler_strategy import AsyncPlaywrightCrawlerStrategy, AsyncHTTPCrawlerStrategy
+
 __all__ = [
+    "Pipeline",
+    "AsyncPlaywrightCrawlerStrategy",
+    "AsyncHTTPCrawlerStrategy",
+    "create_pipeline",
+    "Crawler",
+    "AsyncLoggerBase",
+    "AsyncLogger",
     "AsyncWebCrawler",
+    "BrowserProfiler",
+    "LLMConfig",
+    "DeepCrawlStrategy",
+    "BFSDeepCrawlStrategy",
+    "BestFirstCrawlingStrategy",
+    "DFSDeepCrawlStrategy",
+    "FilterChain",
+    "URLPatternFilter",
+    "ContentTypeFilter",
+    "DomainFilter",
+    "FilterStats",
+    "URLFilter",
+    "SEOFilter",
+    "KeywordRelevanceScorer",
+    "URLScorer",
+    "CompositeScorer",
+    "DomainAuthorityScorer",
+    "FreshnessScorer",
+    "PathDepthScorer",
+    "DeepCrawlDecorator",
     "CrawlResult",
+    "CrawlerHub",
     "CacheMode",
     "ContentScrapingStrategy",
     "WebScrapingStrategy",
     "LXMLWebScrapingStrategy",
     "BrowserConfig",
     "CrawlerRunConfig",
+    "HTTPCrawlerConfig",
     "ExtractionStrategy",
     "LLMExtractionStrategy",
     "CosineStrategy",
     "JsonCssExtractionStrategy",
     "JsonXPathExtractionStrategy",
+    "JsonLxmlExtractionStrategy",
     "ChunkingStrategy",
     "RegexChunking",
     "DefaultMarkdownGenerator",
@@ -55,35 +131,35 @@ __all__ = [
     "CrawlerMonitor",
     "DisplayMode",
     "MarkdownGenerationResult",
+    "Crawl4aiDockerClient",
+    "ProxyRotationStrategy",
+    "RoundRobinProxyStrategy",
 ]
 
 
-def is_sync_version_installed():
-    try:
-        import selenium
+# def is_sync_version_installed():
+#     try:
+#         import selenium # noqa
 
-        return True
-    except ImportError:
-        return False
+#         return True
+#     except ImportError:
+#         return False
 
 
-if is_sync_version_installed():
-    try:
-        from .web_crawler import WebCrawler
+# if is_sync_version_installed():
+#     try:
+#         from .web_crawler import WebCrawler
 
-        __all__.append("WebCrawler")
-    except ImportError:
-        print(
-            "Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies."
-        )
-else:
-    WebCrawler = None
-    # import warnings
-    # print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.")
-
-import warnings
-from pydantic import warnings as pydantic_warnings
+#         __all__.append("WebCrawler")
+#     except ImportError:
+#         print(
+#             "Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies."
+#         )
+# else:
+#     WebCrawler = None
+#     # import warnings
+#     # print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.")
 
 # Disable all Pydantic warnings
 warnings.filterwarnings("ignore", module="pydantic")
-# pydantic_warnings.filter_warnings()
+# pydantic_warnings.filter_warnings()

crawl4ai/__version__.py

@@ -1,2 +1,2 @@
 # crawl4ai/_version.py
-__version__ = "0.4.3b3"
+__version__ = "0.5.0.post8"

crawl4ai/async_configs.py

@@ -1,20 +1,163 @@
+import os
 from .config import (
+    DEFAULT_PROVIDER,
+    DEFAULT_PROVIDER_API_KEY,
     MIN_WORD_THRESHOLD,
     IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
+    PROVIDER_MODELS,
     SCREENSHOT_HEIGHT_TRESHOLD,
     PAGE_TIMEOUT,
     IMAGE_SCORE_THRESHOLD,
     SOCIAL_MEDIA_DOMAINS,
 )
 
-from .user_agent_generator import UserAgentGenerator, UAGen, ValidUAGenerator, OnlineUAGenerator
-from .extraction_strategy import ExtractionStrategy
+from .user_agent_generator import UAGen, ValidUAGenerator  # , OnlineUAGenerator
+from .extraction_strategy import ExtractionStrategy, LLMExtractionStrategy
 from .chunking_strategy import ChunkingStrategy, RegexChunking
-from .markdown_generation_strategy import MarkdownGenerationStrategy
-from .content_filter_strategy import RelevantContentFilter, BM25ContentFilter, LLMContentFilter, PruningContentFilter
+
+from .markdown_generation_strategy import MarkdownGenerationStrategy, DefaultMarkdownGenerator
 from .content_scraping_strategy import ContentScrapingStrategy, WebScrapingStrategy
-from typing import Optional, Union, List
+from .deep_crawling import DeepCrawlStrategy
+
 from .cache_context import CacheMode
+from .proxy_strategy import ProxyRotationStrategy
+
+from typing import Union, List
+import inspect
+from typing import Any, Dict, Optional
+from enum import Enum
+
+from .proxy_strategy import ProxyConfig
+try:
+    from .browser.models import DockerConfig
+except ImportError:
+    DockerConfig = None
+
+
+def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict:
+    """
+    Recursively convert an object to a serializable dictionary using {type, params} structure
+    for complex objects.
+    """
+    if obj is None:
+        return None
+
+    # Handle basic types
+    if isinstance(obj, (str, int, float, bool)):
+        return obj
+
+    # Handle Enum
+    if isinstance(obj, Enum):
+        return {"type": obj.__class__.__name__, "params": obj.value}
+
+    # Handle datetime objects
+    if hasattr(obj, "isoformat"):
+        return obj.isoformat()
+
+    # Handle lists, tuples, and sets, and basically any iterable
+    if isinstance(obj, (list, tuple, set)) or hasattr(obj, '__iter__') and not isinstance(obj, dict):
+        return [to_serializable_dict(item) for item in obj]
+
+    # Handle frozensets, which are not iterable
+    if isinstance(obj, frozenset):
+        return [to_serializable_dict(item) for item in list(obj)]
+
+    # Handle dictionaries - preserve them as-is
+    if isinstance(obj, dict):
+        return {
+            "type": "dict",  # Mark as plain dictionary
+            "value": {str(k): to_serializable_dict(v) for k, v in obj.items()},
+        }
+
+    _type = obj.__class__.__name__
+
+    # Handle class instances
+    if hasattr(obj, "__class__"):
+        # Get constructor signature
+        sig = inspect.signature(obj.__class__.__init__)
+        params = sig.parameters
+
+        # Get current values
+        current_values = {}
+        for name, param in params.items():
+            if name == "self":
+                continue
+
+            value = getattr(obj, name, param.default)
+
+            # Only include if different from default, considering empty values
+            if not (is_empty_value(value) and is_empty_value(param.default)):
+                if value != param.default and not ignore_default_value:
+                    current_values[name] = to_serializable_dict(value)
+        
+        if hasattr(obj, '__slots__'):
+            for slot in obj.__slots__:
+                if slot.startswith('_'):  # Handle private slots
+                    attr_name = slot[1:]  # Remove leading '_'
+                    value = getattr(obj, slot, None)
+                    if value is not None:
+                        current_values[attr_name] = to_serializable_dict(value)
+
+            
+        
+        return {
+            "type": obj.__class__.__name__,
+            "params": current_values
+        }
+        
+    return str(obj)
+
+
+def from_serializable_dict(data: Any) -> Any:
+    """
+    Recursively convert a serializable dictionary back to an object instance.
+    """
+    if data is None:
+        return None
+
+    # Handle basic types
+    if isinstance(data, (str, int, float, bool)):
+        return data
+
+    # Handle typed data
+    if isinstance(data, dict) and "type" in data:
+        # Handle plain dictionaries
+        if data["type"] == "dict":
+            return {k: from_serializable_dict(v) for k, v in data["value"].items()}
+
+        # Import from crawl4ai for class instances
+        import crawl4ai
+
+        cls = getattr(crawl4ai, data["type"])
+
+        # Handle Enum
+        if issubclass(cls, Enum):
+            return cls(data["params"])
+
+        # Handle class instances
+        constructor_args = {
+            k: from_serializable_dict(v) for k, v in data["params"].items()
+        }
+        return cls(**constructor_args)
+
+    # Handle lists
+    if isinstance(data, list):
+        return [from_serializable_dict(item) for item in data]
+
+    # Handle raw dictionaries (legacy support)
+    if isinstance(data, dict):
+        return {k: from_serializable_dict(v) for k, v in data.items()}
+
+    return data
+
+
+def is_empty_value(value: Any) -> bool:
+    """Check if a value is effectively empty/null."""
+    if value is None:
+        return True
+    if isinstance(value, (list, tuple, set, dict, str)) and len(value) == 0:
+        return True
+    return False
 
 
 class BrowserConfig:
@@ -30,6 +173,12 @@ class BrowserConfig:
                             Default: "chromium".
         headless (bool): Whether to run the browser in headless mode (no visible GUI).
                          Default: True.
+        browser_mode (str): Determines how the browser should be initialized:
+                           "builtin" - use the builtin CDP browser running in background
+                           "dedicated" - create a new dedicated browser instance each time
+                           "cdp" - use explicit CDP settings provided in cdp_url
+                           "docker" - run browser in Docker container with isolation
+                           Default: "dedicated"
         use_managed_browser (bool): Launch the browser using a managed approach (e.g., via CDP), allowing
                                     advanced manipulation. Default: False.
         cdp_url (str): URL for the Chrome DevTools Protocol (CDP) endpoint. Default: "ws://localhost:9222/devtools/browser/".
@@ -44,17 +193,21 @@ class BrowserConfig:
                               is "chromium". Default: "chromium".
         proxy (Optional[str]): Proxy server URL (e.g., "http://username:password@proxy:port"). If None, no proxy is used.
                              Default: None.
-        proxy_config (dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
+        proxy_config (ProxyConfig or dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
                                      If None, no additional proxy config. Default: None.
+        docker_config (DockerConfig or dict or None): Configuration for Docker-based browser automation.
+                                     Contains settings for Docker container operation. Default: None.
         viewport_width (int): Default viewport width for pages. Default: 1080.
         viewport_height (int): Default viewport height for pages. Default: 600.
+        viewport (dict): Default viewport dimensions for pages. If set, overrides viewport_width and viewport_height.
+                         Default: None.
         verbose (bool): Enable verbose logging.
                         Default: True.
         accept_downloads (bool): Whether to allow file downloads. If True, requires a downloads_path.
                                  Default: False.
         downloads_path (str or None): Directory to store downloaded files. If None and accept_downloads is True,
                                       a default path will be created. Default: None.
-        storage_state (str or dict or None): Path or object describing storage state (cookies, localStorage).
+        storage_state (str or dict or None): An in-memory storage state (cookies, localStorage).
                                              Default: None.
         ignore_https_errors (bool): Ignore HTTPS certificate errors. Default: True.
         java_script_enabled (bool): Enable JavaScript execution in pages. Default: True.
@@ -80,6 +233,7 @@ class BrowserConfig:
         self,
         browser_type: str = "chromium",
         headless: bool = True,
+        browser_mode: str = "dedicated",
         use_managed_browser: bool = False,
         cdp_url: str = None,
         use_persistent_context: bool = False,
@@ -87,12 +241,14 @@ class BrowserConfig:
         chrome_channel: str = "chromium",
         channel: str = "chromium",
         proxy: str = None,
-        proxy_config: dict = None,
+        proxy_config: Union[ProxyConfig, dict, None] = None,
+        docker_config: Union[DockerConfig, dict, None] = None,
         viewport_width: int = 1080,
         viewport_height: int = 600,
+        viewport: dict = None,
         accept_downloads: bool = False,
         downloads_path: str = None,
-        storage_state : Union[str, dict, None]=None,
+        storage_state: Union[str, dict, None] = None,
         ignore_https_errors: bool = True,
         java_script_enabled: bool = True,
         sleep_on_close: bool = False,
@@ -114,7 +270,8 @@ class BrowserConfig:
         host: str = "localhost",
     ):
         self.browser_type = browser_type
-        self.headless = headless
+        self.headless = headless 
+        self.browser_mode = browser_mode
         self.use_managed_browser = use_managed_browser
         self.cdp_url = cdp_url
         self.use_persistent_context = use_persistent_context
@@ -126,8 +283,22 @@ class BrowserConfig:
             self.chrome_channel = ""
         self.proxy = proxy
         self.proxy_config = proxy_config
+        
+        # Handle docker configuration
+        if isinstance(docker_config, dict) and DockerConfig is not None:
+            self.docker_config = DockerConfig.from_kwargs(docker_config)
+        else:
+            self.docker_config = docker_config
+
+        if self.docker_config:
+            self.user_data_dir = self.docker_config.user_data_dir
+
         self.viewport_width = viewport_width
         self.viewport_height = viewport_height
+        self.viewport = viewport
+        if self.viewport is not None:
+            self.viewport_width = self.viewport.get("width", 1080)
+            self.viewport_height = self.viewport.get("height", 600)
         self.accept_downloads = accept_downloads
         self.downloads_path = downloads_path
         self.storage_state = storage_state
@@ -144,6 +315,7 @@ class BrowserConfig:
         self.sleep_on_close = sleep_on_close
         self.verbose = verbose
         self.debugging_port = debugging_port
+        self.host = host
 
         fa_user_agenr_generator = ValidUAGenerator()
         if self.user_agent_mode == "random":
@@ -152,10 +324,26 @@ class BrowserConfig:
             )
         else:
             pass
-        
+
         self.browser_hint = UAGen.generate_client_hints(self.user_agent)
         self.headers.setdefault("sec-ch-ua", self.browser_hint)
 
+        # Set appropriate browser management flags based on browser_mode
+        if self.browser_mode == "builtin":
+            # Builtin mode uses managed browser connecting to builtin CDP endpoint
+            self.use_managed_browser = True
+            # cdp_url will be set later by browser_manager
+        elif self.browser_mode == "docker":
+            # Docker mode uses managed browser with CDP to connect to browser in container
+            self.use_managed_browser = True
+            # cdp_url will be set later by docker browser strategy
+        elif self.browser_mode == "custom" and self.cdp_url:
+            # Custom mode with explicit CDP URL
+            self.use_managed_browser = True
+        elif self.browser_mode == "dedicated":
+            # Dedicated mode uses a new browser instance each time
+            pass
+
         # If persistent context is requested, ensure managed browser is enabled
         if self.use_persistent_context:
             self.use_managed_browser = True
@@ -165,6 +353,7 @@ class BrowserConfig:
         return BrowserConfig(
             browser_type=kwargs.get("browser_type", "chromium"),
             headless=kwargs.get("headless", True),
+            browser_mode=kwargs.get("browser_mode", "dedicated"),
             use_managed_browser=kwargs.get("use_managed_browser", False),
             cdp_url=kwargs.get("cdp_url"),
             use_persistent_context=kwargs.get("use_persistent_context", False),
@@ -172,7 +361,8 @@ class BrowserConfig:
             chrome_channel=kwargs.get("chrome_channel", "chromium"),
             channel=kwargs.get("channel", "chromium"),
             proxy=kwargs.get("proxy"),
-            proxy_config=kwargs.get("proxy_config"),
+            proxy_config=kwargs.get("proxy_config", None),
+            docker_config=kwargs.get("docker_config", None),
             viewport_width=kwargs.get("viewport_width", 1080),
             viewport_height=kwargs.get("viewport_height", 600),
             accept_downloads=kwargs.get("accept_downloads", False),
@@ -192,12 +382,15 @@ class BrowserConfig:
             text_mode=kwargs.get("text_mode", False),
             light_mode=kwargs.get("light_mode", False),
             extra_args=kwargs.get("extra_args", []),
+            debugging_port=kwargs.get("debugging_port", 9222),
+            host=kwargs.get("host", "localhost"),
         )
 
     def to_dict(self):
-        return {
+        result = {
             "browser_type": self.browser_type,
             "headless": self.headless,
+            "browser_mode": self.browser_mode,
             "use_managed_browser": self.use_managed_browser,
             "cdp_url": self.cdp_url,
             "use_persistent_context": self.use_persistent_context,
@@ -224,14 +417,24 @@ class BrowserConfig:
             "sleep_on_close": self.sleep_on_close,
             "verbose": self.verbose,
             "debugging_port": self.debugging_port,
+            "host": self.host,
         }
+        
+        # Include docker_config if it exists
+        if hasattr(self, "docker_config") and self.docker_config is not None:
+            if hasattr(self.docker_config, "to_dict"):
+                result["docker_config"] = self.docker_config.to_dict()
+            else:
+                result["docker_config"] = self.docker_config
+                
+        return result
 
     def clone(self, **kwargs):
         """Create a copy of this configuration with updated values.
-        
+
         Args:
             **kwargs: Key-value pairs of configuration options to update
-            
+
         Returns:
             BrowserConfig: A new instance with the specified updates
         """
@@ -239,8 +442,98 @@ class BrowserConfig:
         config_dict.update(kwargs)
         return BrowserConfig.from_kwargs(config_dict)
 
+    # Create a funciton returns dict of the object
+    def dump(self) -> dict:
+        # Serialize the object to a dictionary
+        return to_serializable_dict(self)
+
+    @staticmethod
+    def load(data: dict) -> "BrowserConfig":
+        # Deserialize the object from a dictionary
+        config = from_serializable_dict(data)
+        if isinstance(config, BrowserConfig):
+            return config
+        return BrowserConfig.from_kwargs(config)
+
+
+class HTTPCrawlerConfig:
+    """HTTP-specific crawler configuration"""
+
+    method: str = "GET"
+    headers: Optional[Dict[str, str]] = None
+    data: Optional[Dict[str, Any]] = None
+    json: Optional[Dict[str, Any]] = None
+    follow_redirects: bool = True
+    verify_ssl: bool = True
+
+    def __init__(
+        self,
+        method: str = "GET",
+        headers: Optional[Dict[str, str]] = None,
+        data: Optional[Dict[str, Any]] = None,
+        json: Optional[Dict[str, Any]] = None,
+        follow_redirects: bool = True,
+        verify_ssl: bool = True,
+    ):
+        self.method = method
+        self.headers = headers
+        self.data = data
+        self.json = json
+        self.follow_redirects = follow_redirects
+        self.verify_ssl = verify_ssl
+
+    @staticmethod
+    def from_kwargs(kwargs: dict) -> "HTTPCrawlerConfig":
+        return HTTPCrawlerConfig(
+            method=kwargs.get("method", "GET"),
+            headers=kwargs.get("headers"),
+            data=kwargs.get("data"),
+            json=kwargs.get("json"),
+            follow_redirects=kwargs.get("follow_redirects", True),
+            verify_ssl=kwargs.get("verify_ssl", True),
+        )
+
+    def to_dict(self):
+        return {
+            "method": self.method,
+            "headers": self.headers,
+            "data": self.data,
+            "json": self.json,
+            "follow_redirects": self.follow_redirects,
+            "verify_ssl": self.verify_ssl,
+        }
+
+    def clone(self, **kwargs):
+        """Create a copy of this configuration with updated values.
+
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+
+        Returns:
+            HTTPCrawlerConfig: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return HTTPCrawlerConfig.from_kwargs(config_dict)
+
+    def dump(self) -> dict:
+        return to_serializable_dict(self)
+
+    @staticmethod
+    def load(data: dict) -> "HTTPCrawlerConfig":
+        config = from_serializable_dict(data)
+        if isinstance(config, HTTPCrawlerConfig):
+            return config
+        return HTTPCrawlerConfig.from_kwargs(config)
+
+class CrawlerRunConfig():
+    _UNWANTED_PROPS = {
+        'disable_cache' : 'Instead, use cache_mode=CacheMode.DISABLED',
+        'bypass_cache' : 'Instead, use cache_mode=CacheMode.BYPASS',
+        'no_cache_read' : 'Instead, use cache_mode=CacheMode.WRITE_ONLY',
+        'no_cache_write' : 'Instead, use cache_mode=CacheMode.READ_ONLY',
+    }
 
-class CrawlerRunConfig:
     """
     Configuration class for controlling how the crawler runs each crawl operation.
     This includes parameters for content extraction, page manipulation, waiting conditions,
@@ -250,6 +543,9 @@ class CrawlerRunConfig:
     By using this class, you have a single place to understand and adjust the crawling options.
 
     Attributes:
+        # Deep Crawl Parameters
+        deep_crawl_strategy (DeepCrawlStrategy or None): Strategy to use for deep crawling.
+
         # Content Processing Parameters
         word_count_threshold (int): Minimum word count threshold before processing content.
                                     Default: MIN_WORD_THRESHOLD (typically 200).
@@ -259,18 +555,27 @@ class CrawlerRunConfig:
                                               Default: RegexChunking().
         markdown_generator (MarkdownGenerationStrategy): Strategy for generating markdown.
                                                          Default: None.
-        content_filter (RelevantContentFilter or None): Optional filter to prune irrelevant content.
-                                                        Default: None.
         only_text (bool): If True, attempt to extract text-only content where applicable.
                           Default: False.
         css_selector (str or None): CSS selector to extract a specific portion of the page.
                                     Default: None.
+        
+        target_elements (list of str or None): List of CSS selectors for specific elements for Markdown generation 
+                                                and structured data extraction. When you set this, only the contents 
+                                                of these elements are processed for extraction and Markdown generation. 
+                                                If you do not set any value, the entire page is processed. 
+                                                The difference between this and css_selector is that this will shrink 
+                                                the initial raw HTML to the selected element, while this will only affect 
+                                                the extraction and Markdown generation.
+                                    Default: None
         excluded_tags (list of str or None): List of HTML tags to exclude from processing.
                                              Default: None.
         excluded_selector (str or None): CSS selector to exclude from processing.
                                          Default: None.
         keep_data_attributes (bool): If True, retain `data-*` attributes while removing unwanted attributes.
                                      Default: False.
+        keep_attrs (list of str): List of HTML attributes to keep during processing.
+                                      Default: [].
         remove_forms (bool): If True, remove all `<form>` elements from the HTML.
                              Default: False.
         prettiify (bool): If True, apply `fast_format_html` to produce prettified HTML output.
@@ -279,13 +584,15 @@ class CrawlerRunConfig:
                            Default: "lxml".
         scraping_strategy (ContentScrapingStrategy): Scraping strategy to use.
                            Default: WebScrapingStrategy.
-        proxy_config (dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
+        proxy_config (ProxyConfig or dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
                                      If None, no additional proxy config. Default: None.
 
+        # SSL Parameters
+        fetch_ssl_certificate: bool = False,
         # Caching Parameters
         cache_mode (CacheMode or None): Defines how caching is handled.
                                         If None, defaults to CacheMode.ENABLED internally.
-                                        Default: None.
+                                        Default: CacheMode.BYPASS.
         session_id (str or None): Optional session ID to persist the browser context and the created
                                   page instance. If the ID already exists, the crawler does not
                                   create a new page and uses the current page to preserve the state.
@@ -357,16 +664,22 @@ class CrawlerRunConfig:
                                      Default: IMAGE_SCORE_THRESHOLD (e.g., 3).
         exclude_external_images (bool): If True, exclude all external images from processing.
                                          Default: False.
+        table_score_threshold (int): Minimum score threshold for processing a table.
+                                     Default: 7.
 
         # Link and Domain Handling Parameters
         exclude_social_media_domains (list of str): List of domains to exclude for social media links.
                                                     Default: SOCIAL_MEDIA_DOMAINS (from config).
         exclude_external_links (bool): If True, exclude all external links from the results.
                                        Default: False.
+        exclude_internal_links (bool): If True, exclude internal links from the results.
+                                       Default: False.
         exclude_social_media_links (bool): If True, exclude links pointing to social media domains.
                                            Default: False.
         exclude_domains (list of str): List of specific domains to exclude from results.
                                        Default: [].
+        exclude_internal_links (bool): If True, exclude internal links from the results.
+                                       Default: False.
 
         # Debugging and Logging Parameters
         verbose (bool): Enable verbose logging.
@@ -374,19 +687,33 @@ class CrawlerRunConfig:
         log_console (bool): If True, log console messages from the page.
                             Default: False.
 
-        # Streaming Parameters
+        # HTTP Crwler Strategy Parameters
+        method (str): HTTP method to use for the request, when using AsyncHTTPCrwalerStrategy.
+                        Default: "GET".
+        data (dict): Data to send in the request body, when using AsyncHTTPCrwalerStrategy.
+                        Default: None.
+        json (dict): JSON data to send in the request body, when using AsyncHTTPCrwalerStrategy.
+
+        # Connection Parameters
         stream (bool): If True, enables streaming of crawled URLs as they are processed when used with arun_many.
                       Default: False.
 
-        # Optional Parameters
-        stream (bool): If True, stream the page content as it is being loaded.
-        url: str = None  # This is not a compulsory parameter
         check_robots_txt (bool): Whether to check robots.txt rules before crawling. Default: False
-        user_agent (str): Custom User-Agent string to use. Default: None
-        user_agent_mode (str or None): Mode for generating the user agent (e.g., "random"). If None, use the provided
-                                       user_agent as-is. Default: None.
+                                 Default: False.
+        user_agent (str): Custom User-Agent string to use.
+                          Default: None.
+        user_agent_mode (str or None): Mode for generating the user agent (e.g., "random"). If None, use the provided user_agent as-is.
+                                       Default: None.
         user_agent_generator_config (dict or None): Configuration for user agent generation if user_agent_mode is set.
                                                     Default: None.
+
+        # Experimental Parameters
+        experimental (dict): Dictionary containing experimental parameters that are in beta phase.
+                            This allows passing temporary features that are not yet fully integrated 
+                            into the main parameter set.
+                            Default: None.
+
+        url: str = None  # This is not a compulsory parameter
     """
 
     def __init__(
@@ -395,22 +722,24 @@ class CrawlerRunConfig:
         word_count_threshold: int = MIN_WORD_THRESHOLD,
         extraction_strategy: ExtractionStrategy = None,
         chunking_strategy: ChunkingStrategy = RegexChunking(),
-        markdown_generator: MarkdownGenerationStrategy = None,
-        content_filter : RelevantContentFilter = None,
+        markdown_generator: MarkdownGenerationStrategy = DefaultMarkdownGenerator(),
         only_text: bool = False,
         css_selector: str = None,
+        target_elements: List[str] = None,
         excluded_tags: list = None,
         excluded_selector: str = None,
         keep_data_attributes: bool = False,
+        keep_attrs: list = None,
         remove_forms: bool = False,
         prettiify: bool = False,
         parser_type: str = "lxml",
         scraping_strategy: ContentScrapingStrategy = None,
-        proxy_config: dict = None,
+        proxy_config: Union[ProxyConfig, dict, None] = None,
+        proxy_rotation_strategy: Optional[ProxyRotationStrategy] = None,
         # SSL Parameters
         fetch_ssl_certificate: bool = False,
         # Caching Parameters
-        cache_mode: CacheMode =None,
+        cache_mode: CacheMode = CacheMode.BYPASS,
         session_id: str = None,
         bypass_cache: bool = False,
         disable_cache: bool = False,
@@ -445,23 +774,31 @@ class CrawlerRunConfig:
         pdf: bool = False,
         image_description_min_word_threshold: int = IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
         image_score_threshold: int = IMAGE_SCORE_THRESHOLD,
+        table_score_threshold: int = 7,
         exclude_external_images: bool = False,
         # Link and Domain Handling Parameters
         exclude_social_media_domains: list = None,
         exclude_external_links: bool = False,
         exclude_social_media_links: bool = False,
         exclude_domains: list = None,
+        exclude_internal_links: bool = False,
         # Debugging and Logging Parameters
         verbose: bool = True,
         log_console: bool = False,
-        # Streaming Parameters
+        # Connection Parameters
+        method: str = "GET",
         stream: bool = False,
         url: str = None,
         check_robots_txt: bool = False,
         user_agent: str = None,
         user_agent_mode: str = None,
         user_agent_generator_config: dict = {},
+        # Deep Crawl Parameters
+        deep_crawl_strategy: Optional[DeepCrawlStrategy] = None,
+        # Experimental Parameters
+        experimental: Dict[str, Any] = None,
     ):
+        # TODO: Planning to set properties dynamically based on the __init__ signature
         self.url = url
 
         # Content Processing Parameters
@@ -469,17 +806,19 @@ class CrawlerRunConfig:
         self.extraction_strategy = extraction_strategy
         self.chunking_strategy = chunking_strategy
         self.markdown_generator = markdown_generator
-        self.content_filter = content_filter
         self.only_text = only_text
         self.css_selector = css_selector
+        self.target_elements = target_elements or []
         self.excluded_tags = excluded_tags or []
         self.excluded_selector = excluded_selector or ""
         self.keep_data_attributes = keep_data_attributes
+        self.keep_attrs = keep_attrs or []
         self.remove_forms = remove_forms
         self.prettiify = prettiify
         self.parser_type = parser_type
         self.scraping_strategy = scraping_strategy or WebScrapingStrategy()
         self.proxy_config = proxy_config
+        self.proxy_rotation_strategy = proxy_rotation_strategy
 
         # SSL Parameters
         self.fetch_ssl_certificate = fetch_ssl_certificate
@@ -524,6 +863,7 @@ class CrawlerRunConfig:
         self.image_description_min_word_threshold = image_description_min_word_threshold
         self.image_score_threshold = image_score_threshold
         self.exclude_external_images = exclude_external_images
+        self.table_score_threshold = table_score_threshold
 
         # Link and Domain Handling Parameters
         self.exclude_social_media_domains = (
@@ -532,13 +872,15 @@ class CrawlerRunConfig:
         self.exclude_external_links = exclude_external_links
         self.exclude_social_media_links = exclude_social_media_links
         self.exclude_domains = exclude_domains or []
+        self.exclude_internal_links = exclude_internal_links
 
         # Debugging and Logging Parameters
         self.verbose = verbose
         self.log_console = log_console
 
-        # Streaming Parameters
+        # Connection Parameters
         self.stream = stream
+        self.method = method
 
         # Robots.txt Handling Parameters
         self.check_robots_txt = check_robots_txt
@@ -566,6 +908,30 @@ class CrawlerRunConfig:
         if self.chunking_strategy is None:
             self.chunking_strategy = RegexChunking()
 
+        # Deep Crawl Parameters
+        self.deep_crawl_strategy = deep_crawl_strategy
+        
+        # Experimental Parameters
+        self.experimental = experimental or {}
+
+
+    def __getattr__(self, name):
+        """Handle attribute access."""
+        if name in self._UNWANTED_PROPS:
+            raise AttributeError(f"Getting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}")
+        raise AttributeError(f"'{self.__class__.__name__}' has no attribute '{name}'")
+
+    def __setattr__(self, name, value):
+        """Handle attribute setting."""
+        # TODO: Planning to set properties dynamically based on the __init__ signature
+        sig = inspect.signature(self.__init__)
+        all_params = sig.parameters  # Dictionary of parameter names and their details
+
+        if name in self._UNWANTED_PROPS and value is not all_params[name].default:
+            raise AttributeError(f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}")
+        
+        super().__setattr__(name, value)
+
     @staticmethod
     def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
         return CrawlerRunConfig(
@@ -574,21 +940,23 @@ class CrawlerRunConfig:
             extraction_strategy=kwargs.get("extraction_strategy"),
             chunking_strategy=kwargs.get("chunking_strategy", RegexChunking()),
             markdown_generator=kwargs.get("markdown_generator"),
-            content_filter=kwargs.get("content_filter"),
             only_text=kwargs.get("only_text", False),
             css_selector=kwargs.get("css_selector"),
+            target_elements=kwargs.get("target_elements", []),
             excluded_tags=kwargs.get("excluded_tags", []),
             excluded_selector=kwargs.get("excluded_selector", ""),
             keep_data_attributes=kwargs.get("keep_data_attributes", False),
+            keep_attrs=kwargs.get("keep_attrs", []),
             remove_forms=kwargs.get("remove_forms", False),
             prettiify=kwargs.get("prettiify", False),
             parser_type=kwargs.get("parser_type", "lxml"),
             scraping_strategy=kwargs.get("scraping_strategy"),
             proxy_config=kwargs.get("proxy_config"),
+            proxy_rotation_strategy=kwargs.get("proxy_rotation_strategy"),
             # SSL Parameters
             fetch_ssl_certificate=kwargs.get("fetch_ssl_certificate", False),
             # Caching Parameters
-            cache_mode=kwargs.get("cache_mode"),
+            cache_mode=kwargs.get("cache_mode", CacheMode.BYPASS),
             session_id=kwargs.get("session_id"),
             bypass_cache=kwargs.get("bypass_cache", False),
             disable_cache=kwargs.get("disable_cache", False),
@@ -630,6 +998,7 @@ class CrawlerRunConfig:
             image_score_threshold=kwargs.get(
                 "image_score_threshold", IMAGE_SCORE_THRESHOLD
             ),
+            table_score_threshold=kwargs.get("table_score_threshold", 7),
             exclude_external_images=kwargs.get("exclude_external_images", False),
             # Link and Domain Handling Parameters
             exclude_social_media_domains=kwargs.get(
@@ -638,36 +1007,56 @@ class CrawlerRunConfig:
             exclude_external_links=kwargs.get("exclude_external_links", False),
             exclude_social_media_links=kwargs.get("exclude_social_media_links", False),
             exclude_domains=kwargs.get("exclude_domains", []),
+            exclude_internal_links=kwargs.get("exclude_internal_links", False),
             # Debugging and Logging Parameters
             verbose=kwargs.get("verbose", True),
             log_console=kwargs.get("log_console", False),
-            # Streaming Parameters
+            # Connection Parameters
+            method=kwargs.get("method", "GET"),
             stream=kwargs.get("stream", False),
-            url=kwargs.get("url"),
             check_robots_txt=kwargs.get("check_robots_txt", False),
             user_agent=kwargs.get("user_agent"),
             user_agent_mode=kwargs.get("user_agent_mode"),
             user_agent_generator_config=kwargs.get("user_agent_generator_config", {}),
+            # Deep Crawl Parameters
+            deep_crawl_strategy=kwargs.get("deep_crawl_strategy"),
+            url=kwargs.get("url"),
+            # Experimental Parameters 
+            experimental=kwargs.get("experimental"),
         )
 
     # Create a funciton returns dict of the object
+    def dump(self) -> dict:
+        # Serialize the object to a dictionary
+        return to_serializable_dict(self)
+
+    @staticmethod
+    def load(data: dict) -> "CrawlerRunConfig":
+        # Deserialize the object from a dictionary
+        config = from_serializable_dict(data)
+        if isinstance(config, CrawlerRunConfig):
+            return config
+        return CrawlerRunConfig.from_kwargs(config)
+
     def to_dict(self):
         return {
             "word_count_threshold": self.word_count_threshold,
             "extraction_strategy": self.extraction_strategy,
             "chunking_strategy": self.chunking_strategy,
             "markdown_generator": self.markdown_generator,
-            "content_filter": self.content_filter,
             "only_text": self.only_text,
             "css_selector": self.css_selector,
+            "target_elements": self.target_elements,
             "excluded_tags": self.excluded_tags,
             "excluded_selector": self.excluded_selector,
             "keep_data_attributes": self.keep_data_attributes,
+            "keep_attrs": self.keep_attrs,
             "remove_forms": self.remove_forms,
             "prettiify": self.prettiify,
             "parser_type": self.parser_type,
             "scraping_strategy": self.scraping_strategy,
             "proxy_config": self.proxy_config,
+            "proxy_rotation_strategy": self.proxy_rotation_strategy,
             "fetch_ssl_certificate": self.fetch_ssl_certificate,
             "cache_mode": self.cache_mode,
             "session_id": self.session_id,
@@ -701,35 +1090,40 @@ class CrawlerRunConfig:
             "pdf": self.pdf,
             "image_description_min_word_threshold": self.image_description_min_word_threshold,
             "image_score_threshold": self.image_score_threshold,
+            "table_score_threshold": self.table_score_threshold,
             "exclude_external_images": self.exclude_external_images,
             "exclude_social_media_domains": self.exclude_social_media_domains,
             "exclude_external_links": self.exclude_external_links,
             "exclude_social_media_links": self.exclude_social_media_links,
             "exclude_domains": self.exclude_domains,
+            "exclude_internal_links": self.exclude_internal_links,
             "verbose": self.verbose,
             "log_console": self.log_console,
+            "method": self.method,
             "stream": self.stream,
-            "url": self.url,
             "check_robots_txt": self.check_robots_txt,
             "user_agent": self.user_agent,
             "user_agent_mode": self.user_agent_mode,
             "user_agent_generator_config": self.user_agent_generator_config,
+            "deep_crawl_strategy": self.deep_crawl_strategy,
+            "url": self.url,
+            "experimental": self.experimental,
         }
 
     def clone(self, **kwargs):
         """Create a copy of this configuration with updated values.
-        
+
         Args:
             **kwargs: Key-value pairs of configuration options to update
-            
+
         Returns:
             CrawlerRunConfig: A new instance with the specified updates
-            
+
         Example:
             ```python
             # Create a new config with streaming enabled
             stream_config = config.clone(stream=True)
-            
+
             # Create a new config with multiple updates
             new_config = config.clone(
                 stream=True,
@@ -741,3 +1135,81 @@ class CrawlerRunConfig:
         config_dict = self.to_dict()
         config_dict.update(kwargs)
         return CrawlerRunConfig.from_kwargs(config_dict)
+
+
+class LLMConfig:
+    def __init__(
+        self,
+        provider: str = DEFAULT_PROVIDER,
+        api_token: Optional[str] = None,
+        base_url: Optional[str] = None,
+        temprature: Optional[float] = None,
+        max_tokens: Optional[int] = None,
+        top_p: Optional[float] = None,
+        frequency_penalty: Optional[float] = None,
+        presence_penalty: Optional[float] = None,
+        stop: Optional[List[str]] = None,
+        n: Optional[int] = None,    
+    ):
+        """Configuaration class for LLM provider and API token."""
+        self.provider = provider
+        if api_token and not api_token.startswith("env:"):
+            self.api_token = api_token
+        elif api_token and api_token.startswith("env:"):
+            self.api_token = os.getenv(api_token[4:])
+        else:
+            self.api_token = PROVIDER_MODELS.get(provider, "no-token") or os.getenv(
+                DEFAULT_PROVIDER_API_KEY
+            )
+        self.base_url = base_url
+        self.temprature = temprature
+        self.max_tokens = max_tokens
+        self.top_p = top_p
+        self.frequency_penalty = frequency_penalty
+        self.presence_penalty = presence_penalty
+        self.stop = stop
+        self.n = n
+
+    @staticmethod
+    def from_kwargs(kwargs: dict) -> "LLMConfig":
+        return LLMConfig(
+            provider=kwargs.get("provider", DEFAULT_PROVIDER),
+            api_token=kwargs.get("api_token"),
+            base_url=kwargs.get("base_url"),
+            temprature=kwargs.get("temprature"),
+            max_tokens=kwargs.get("max_tokens"),
+            top_p=kwargs.get("top_p"),
+            frequency_penalty=kwargs.get("frequency_penalty"),
+            presence_penalty=kwargs.get("presence_penalty"),
+            stop=kwargs.get("stop"),
+            n=kwargs.get("n")
+        )
+
+    def to_dict(self):
+        return {
+            "provider": self.provider,
+            "api_token": self.api_token,
+            "base_url": self.base_url,
+            "temprature": self.temprature,
+            "max_tokens": self.max_tokens,
+            "top_p": self.top_p,
+            "frequency_penalty": self.frequency_penalty,
+            "presence_penalty": self.presence_penalty,
+            "stop": self.stop,
+            "n": self.n
+        }
+
+    def clone(self, **kwargs):
+        """Create a copy of this configuration with updated values.
+
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+
+        Returns:
+            llm_config: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return LLMConfig.from_kwargs(config_dict)
+
+

crawl4ai/async_database.py

@@ -4,19 +4,14 @@ import aiosqlite
 import asyncio
 from typing import Optional, Dict
 from contextlib import asynccontextmanager
-import logging
-import json  # Added for serialization/deserialization
-from .utils import ensure_content_dirs, generate_content_hash
-from .models import CrawlResult, MarkdownGenerationResult
+import json  
+from .models import CrawlResult, MarkdownGenerationResult, StringCompatibleMarkdown
 import aiofiles
-from .version_manager import VersionManager
 from .async_logger import AsyncLogger
-from .utils import get_error_context, create_box_message
 
-# Set up logging
-# logging.basicConfig(level=logging.INFO)
-# logger = logging.getLogger(__name__)
-# logger.setLevel(logging.INFO)
+from .utils import ensure_content_dirs, generate_content_hash
+from .utils import VersionManager
+from .utils import get_error_context, create_box_message
 
 base_directory = DB_PATH = os.path.join(
     os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
@@ -336,12 +331,17 @@ class AsyncDatabaseManager:
                     except json.JSONDecodeError:
                         # Very UGLY, never mention it to me please
                         if field == "markdown" and isinstance(row_dict[field], str):
-                            row_dict[field] = row_dict[field]
+                            row_dict[field] = MarkdownGenerationResult(
+                                raw_markdown=row_dict[field] or "",
+                                markdown_with_citations="",
+                                references_markdown="",
+                                fit_markdown="",
+                                fit_html="",
+                            )
                         else:
                             row_dict[field] = {}
 
                 if isinstance(row_dict["markdown"], Dict):
-                    row_dict["markdown_v2"] = row_dict["markdown"]
                     if row_dict["markdown"].get("raw_markdown"):
                         row_dict["markdown"] = row_dict["markdown"]["raw_markdown"]
 
@@ -358,7 +358,7 @@ class AsyncDatabaseManager:
                 # Remove any fields not in CrawlResult model
                 valid_fields = CrawlResult.__annotations__.keys()
                 filtered_dict = {k: v for k, v in row_dict.items() if k in valid_fields}
-
+                filtered_dict["markdown"] = row_dict["markdown"]
                 return CrawlResult(**filtered_dict)
 
         try:
@@ -384,14 +384,14 @@ class AsyncDatabaseManager:
         }
 
         try:
-            if isinstance(result.markdown, MarkdownGenerationResult):
+            if isinstance(result.markdown, StringCompatibleMarkdown):
                 content_map["markdown"] = (
-                    result.markdown.model_dump_json(),
+                    result.markdown,
                     "markdown",
                 )
-            elif hasattr(result, "markdown_v2"):
+            elif isinstance(result.markdown, MarkdownGenerationResult):
                 content_map["markdown"] = (
-                    result.markdown_v2.model_dump_json(),
+                    result.markdown.model_dump_json(),
                     "markdown",
                 )
             elif isinstance(result.markdown, str):

crawl4ai/async_dispatcher.py

@@ -4,17 +4,15 @@ from .models import (
     CrawlResult,
     CrawlerTaskResult,
     CrawlStatus,
-    DisplayMode,
-    CrawlStats,
     DomainState,
 )
 
-from rich.live import Live
-from rich.table import Table
-from rich.console import Console
-from rich import box
-from datetime import datetime, timedelta
+from .components.crawler_monitor import CrawlerMonitor
+
+from .types import AsyncWebCrawler
+
 from collections.abc import AsyncGenerator
+
 import time
 import psutil
 import asyncio
@@ -25,7 +23,6 @@ import random
 from abc import ABC, abstractmethod
 
 
-
 class RateLimiter:
     def __init__(
         self,
@@ -86,201 +83,6 @@ class RateLimiter:
         return True
 
 
-class CrawlerMonitor:
-    def __init__(
-        self,
-        max_visible_rows: int = 15,
-        display_mode: DisplayMode = DisplayMode.DETAILED,
-    ):
-        self.console = Console()
-        self.max_visible_rows = max_visible_rows
-        self.display_mode = display_mode
-        self.stats: Dict[str, CrawlStats] = {}
-        self.process = psutil.Process()
-        self.start_time = datetime.now()
-        self.live = Live(self._create_table(), refresh_per_second=2)
-
-    def start(self):
-        self.live.start()
-
-    def stop(self):
-        self.live.stop()
-
-    def add_task(self, task_id: str, url: str):
-        self.stats[task_id] = CrawlStats(
-            task_id=task_id, url=url, status=CrawlStatus.QUEUED
-        )
-        self.live.update(self._create_table())
-
-    def update_task(self, task_id: str, **kwargs):
-        if task_id in self.stats:
-            for key, value in kwargs.items():
-                setattr(self.stats[task_id], key, value)
-            self.live.update(self._create_table())
-
-    def _create_aggregated_table(self) -> Table:
-        """Creates a compact table showing only aggregated statistics"""
-        table = Table(
-            box=box.ROUNDED,
-            title="Crawler Status Overview",
-            title_style="bold magenta",
-            header_style="bold blue",
-            show_lines=True,
-        )
-
-        # Calculate statistics
-        total_tasks = len(self.stats)
-        queued = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.QUEUED
-        )
-        in_progress = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.IN_PROGRESS
-        )
-        completed = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.COMPLETED
-        )
-        failed = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.FAILED
-        )
-
-        # Memory statistics
-        current_memory = self.process.memory_info().rss / (1024 * 1024)
-        total_task_memory = sum(stat.memory_usage for stat in self.stats.values())
-        peak_memory = max(
-            (stat.peak_memory for stat in self.stats.values()), default=0.0
-        )
-
-        # Duration
-        duration = datetime.now() - self.start_time
-
-        # Create status row
-        table.add_column("Status", style="bold cyan")
-        table.add_column("Count", justify="right")
-        table.add_column("Percentage", justify="right")
-
-        table.add_row("Total Tasks", str(total_tasks), "100%")
-        table.add_row(
-            "[yellow]In Queue[/yellow]",
-            str(queued),
-            f"{(queued/total_tasks*100):.1f}%" if total_tasks > 0 else "0%",
-        )
-        table.add_row(
-            "[blue]In Progress[/blue]",
-            str(in_progress),
-            f"{(in_progress/total_tasks*100):.1f}%" if total_tasks > 0 else "0%",
-        )
-        table.add_row(
-            "[green]Completed[/green]",
-            str(completed),
-            f"{(completed/total_tasks*100):.1f}%" if total_tasks > 0 else "0%",
-        )
-        table.add_row(
-            "[red]Failed[/red]",
-            str(failed),
-            f"{(failed/total_tasks*100):.1f}%" if total_tasks > 0 else "0%",
-        )
-
-        # Add memory information
-        table.add_section()
-        table.add_row(
-            "[magenta]Current Memory[/magenta]", f"{current_memory:.1f} MB", ""
-        )
-        table.add_row(
-            "[magenta]Total Task Memory[/magenta]", f"{total_task_memory:.1f} MB", ""
-        )
-        table.add_row(
-            "[magenta]Peak Task Memory[/magenta]", f"{peak_memory:.1f} MB", ""
-        )
-        table.add_row(
-            "[yellow]Runtime[/yellow]",
-            str(timedelta(seconds=int(duration.total_seconds()))),
-            "",
-        )
-
-        return table
-
-    def _create_detailed_table(self) -> Table:
-        table = Table(
-            box=box.ROUNDED,
-            title="Crawler Performance Monitor",
-            title_style="bold magenta",
-            header_style="bold blue",
-        )
-
-        # Add columns
-        table.add_column("Task ID", style="cyan", no_wrap=True)
-        table.add_column("URL", style="cyan", no_wrap=True)
-        table.add_column("Status", style="bold")
-        table.add_column("Memory (MB)", justify="right")
-        table.add_column("Peak (MB)", justify="right")
-        table.add_column("Duration", justify="right")
-        table.add_column("Info", style="italic")
-
-        # Add summary row
-        total_memory = sum(stat.memory_usage for stat in self.stats.values())
-        active_count = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.IN_PROGRESS
-        )
-        completed_count = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.COMPLETED
-        )
-        failed_count = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.FAILED
-        )
-
-        table.add_row(
-            "[bold yellow]SUMMARY",
-            f"Total: {len(self.stats)}",
-            f"Active: {active_count}",
-            f"{total_memory:.1f}",
-            f"{self.process.memory_info().rss / (1024 * 1024):.1f}",
-            str(
-                timedelta(
-                    seconds=int((datetime.now() - self.start_time).total_seconds())
-                )
-            ),
-            f"✓{completed_count} ✗{failed_count}",
-            style="bold",
-        )
-
-        table.add_section()
-
-        # Add rows for each task
-        visible_stats = sorted(
-            self.stats.values(),
-            key=lambda x: (
-                x.status != CrawlStatus.IN_PROGRESS,
-                x.status != CrawlStatus.QUEUED,
-                x.end_time or datetime.max,
-            ),
-        )[: self.max_visible_rows]
-
-        for stat in visible_stats:
-            status_style = {
-                CrawlStatus.QUEUED: "white",
-                CrawlStatus.IN_PROGRESS: "yellow",
-                CrawlStatus.COMPLETED: "green",
-                CrawlStatus.FAILED: "red",
-            }[stat.status]
-
-            table.add_row(
-                stat.task_id[:8],  # Show first 8 chars of task ID
-                stat.url[:40] + "..." if len(stat.url) > 40 else stat.url,
-                f"[{status_style}]{stat.status.value}[/{status_style}]",
-                f"{stat.memory_usage:.1f}",
-                f"{stat.peak_memory:.1f}",
-                stat.duration,
-                stat.error_message[:40] if stat.error_message else "",
-            )
-
-        return table
-
-    def _create_table(self) -> Table:
-        """Creates the appropriate table based on display mode"""
-        if self.display_mode == DisplayMode.AGGREGATED:
-            return self._create_aggregated_table()
-        return self._create_detailed_table()
-
 
 class BaseDispatcher(ABC):
     def __init__(
@@ -308,7 +110,7 @@ class BaseDispatcher(ABC):
     async def run_urls(
         self,
         urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821
+        crawler: AsyncWebCrawler,  # noqa: F821
         config: CrawlerRunConfig,
         monitor: Optional[CrawlerMonitor] = None,
     ) -> List[CrawlerTaskResult]:
@@ -319,71 +121,144 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
     def __init__(
         self,
         memory_threshold_percent: float = 90.0,
+        critical_threshold_percent: float = 95.0,  # New critical threshold
+        recovery_threshold_percent: float = 85.0,  # New recovery threshold
         check_interval: float = 1.0,
         max_session_permit: int = 20,
-        memory_wait_timeout: float = 300.0,  # 5 minutes default timeout
+        fairness_timeout: float = 600.0,  # 10 minutes before prioritizing long-waiting URLs
         rate_limiter: Optional[RateLimiter] = None,
         monitor: Optional[CrawlerMonitor] = None,
     ):
         super().__init__(rate_limiter, monitor)
         self.memory_threshold_percent = memory_threshold_percent
+        self.critical_threshold_percent = critical_threshold_percent
+        self.recovery_threshold_percent = recovery_threshold_percent
         self.check_interval = check_interval
         self.max_session_permit = max_session_permit
-        self.memory_wait_timeout = memory_wait_timeout
-        self.result_queue = asyncio.Queue()  # Queue for storing results
-
+        self.fairness_timeout = fairness_timeout
+        self.result_queue = asyncio.Queue()
+        self.task_queue = asyncio.PriorityQueue()  # Priority queue for better management
+        self.memory_pressure_mode = False  # Flag to indicate when we're in memory pressure mode
+        self.current_memory_percent = 0.0  # Track current memory usage
+        
+    async def _memory_monitor_task(self):
+        """Background task to continuously monitor memory usage and update state"""
+        while True:
+            self.current_memory_percent = psutil.virtual_memory().percent
+            
+            # Enter memory pressure mode if we cross the threshold
+            if not self.memory_pressure_mode and self.current_memory_percent >= self.memory_threshold_percent:
+                self.memory_pressure_mode = True
+                if self.monitor:
+                    self.monitor.update_memory_status("PRESSURE")
+            
+            # Exit memory pressure mode if we go below recovery threshold
+            elif self.memory_pressure_mode and self.current_memory_percent <= self.recovery_threshold_percent:
+                self.memory_pressure_mode = False
+                if self.monitor:
+                    self.monitor.update_memory_status("NORMAL")
+            
+            # In critical mode, we might need to take more drastic action
+            if self.current_memory_percent >= self.critical_threshold_percent:
+                if self.monitor:
+                    self.monitor.update_memory_status("CRITICAL")
+                # We could implement additional memory-saving measures here
+                
+            await asyncio.sleep(self.check_interval)
+    
+    def _get_priority_score(self, wait_time: float, retry_count: int) -> float:
+        """Calculate priority score (lower is higher priority)
+        - URLs waiting longer than fairness_timeout get higher priority
+        - More retry attempts decreases priority
+        """
+        if wait_time > self.fairness_timeout:
+            # High priority for long-waiting URLs
+            return -wait_time
+        # Standard priority based on retries
+        return retry_count
+    
     async def crawl_url(
         self,
         url: str,
         config: CrawlerRunConfig,
         task_id: str,
+        retry_count: int = 0,
     ) -> CrawlerTaskResult:
-        start_time = datetime.now()
+        start_time = time.time()
         error_message = ""
         memory_usage = peak_memory = 0.0
-
+        
+        # Get starting memory for accurate measurement
+        process = psutil.Process()
+        start_memory = process.memory_info().rss / (1024 * 1024)
+        
         try:
             if self.monitor:
                 self.monitor.update_task(
-                    task_id, status=CrawlStatus.IN_PROGRESS, start_time=start_time
+                    task_id, 
+                    status=CrawlStatus.IN_PROGRESS, 
+                    start_time=start_time,
+                    retry_count=retry_count
                 )
+                
             self.concurrent_sessions += 1
-
+            
             if self.rate_limiter:
                 await self.rate_limiter.wait_if_needed(url)
-
-            process = psutil.Process()
-            start_memory = process.memory_info().rss / (1024 * 1024)
+                
+            # Check if we're in critical memory state
+            if self.current_memory_percent >= self.critical_threshold_percent:
+                # Requeue this task with increased priority and retry count
+                enqueue_time = time.time()
+                priority = self._get_priority_score(enqueue_time - start_time, retry_count + 1)
+                await self.task_queue.put((priority, (url, task_id, retry_count + 1, enqueue_time)))
+                
+                # Update monitoring
+                if self.monitor:
+                    self.monitor.update_task(
+                        task_id,
+                        status=CrawlStatus.QUEUED,
+                        error_message="Requeued due to critical memory pressure"
+                    )
+                
+                # Return placeholder result with requeued status
+                return CrawlerTaskResult(
+                    task_id=task_id,
+                    url=url,
+                    result=CrawlResult(
+                        url=url, html="", metadata={"status": "requeued"}, 
+                        success=False, error_message="Requeued due to critical memory pressure"
+                    ),
+                    memory_usage=0,
+                    peak_memory=0,
+                    start_time=start_time,
+                    end_time=time.time(),
+                    error_message="Requeued due to critical memory pressure",
+                    retry_count=retry_count + 1
+                )
+            
+            # Execute the crawl
             result = await self.crawler.arun(url, config=config, session_id=task_id)
+            
+            # Measure memory usage
             end_memory = process.memory_info().rss / (1024 * 1024)
-
             memory_usage = peak_memory = end_memory - start_memory
-
+            
+            # Handle rate limiting
             if self.rate_limiter and result.status_code:
                 if not self.rate_limiter.update_delay(url, result.status_code):
                     error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
                     if self.monitor:
                         self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                    result = CrawlerTaskResult(
-                        task_id=task_id,
-                        url=url,
-                        result=result,
-                        memory_usage=memory_usage,
-                        peak_memory=peak_memory,
-                        start_time=start_time,
-                        end_time=datetime.now(),
-                        error_message=error_message,
-                    )
-                    await self.result_queue.put(result)
-                    return result
-
+                        
+            # Update status based on result
             if not result.success:
                 error_message = result.error_message
                 if self.monitor:
                     self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
             elif self.monitor:
                 self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
-
+                
         except Exception as e:
             error_message = str(e)
             if self.monitor:
@@ -391,9 +266,9 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             result = CrawlResult(
                 url=url, html="", metadata={}, success=False, error_message=str(e)
             )
-
+            
         finally:
-            end_time = datetime.now()
+            end_time = time.time()
             if self.monitor:
                 self.monitor.update_task(
                     task_id,
@@ -401,9 +276,10 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
                     memory_usage=memory_usage,
                     peak_memory=peak_memory,
                     error_message=error_message,
+                    retry_count=retry_count
                 )
             self.concurrent_sessions -= 1
-
+            
         return CrawlerTaskResult(
             task_id=task_id,
             url=url,
@@ -413,117 +289,240 @@ class MemoryAdaptiveDispatcher(BaseDispatcher):
             start_time=start_time,
             end_time=end_time,
             error_message=error_message,
+            retry_count=retry_count
         )
-
+        
     async def run_urls(
         self,
         urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821
+        crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
-        ) -> List[CrawlerTaskResult]:
-            self.crawler = crawler
-
-            if self.monitor:
-                self.monitor.start()
-
-            try:
-                pending_tasks = []
-                active_tasks = []
-                task_queue = []
-
-                for url in urls:
-                    task_id = str(uuid.uuid4())
-                    if self.monitor:
-                        self.monitor.add_task(task_id, url)
-                    task_queue.append((url, task_id))
-
-                while task_queue or active_tasks:
-                    wait_start_time = time.time()
-                    while len(active_tasks) < self.max_session_permit and task_queue:
-                        if psutil.virtual_memory().percent >= self.memory_threshold_percent:
-                            # Check if we've exceeded the timeout
-                            if time.time() - wait_start_time > self.memory_wait_timeout:
-                                raise MemoryError(
-                                    f"Memory usage above threshold ({self.memory_threshold_percent}%) for more than {self.memory_wait_timeout} seconds"
-                                )
-                            await asyncio.sleep(self.check_interval)
-                            continue
-
-                        url, task_id = task_queue.pop(0)
-                        task = asyncio.create_task(self.crawl_url(url, config, task_id))
-                        active_tasks.append(task)
-
-                    if not active_tasks:
-                        await asyncio.sleep(self.check_interval)
-                        continue
-
-                    done, pending = await asyncio.wait(
-                        active_tasks, return_when=asyncio.FIRST_COMPLETED
-                    )
-
-                    pending_tasks.extend(done)
-                    active_tasks = list(pending)
-
-                return await asyncio.gather(*pending_tasks)
-            finally:
-                if self.monitor:
-                    self.monitor.stop()
-
-    async def run_urls_stream(
-        self,
-        urls: List[str],
-        crawler: "AsyncWebCrawler",
-        config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlerTaskResult, None]:
+    ) -> List[CrawlerTaskResult]:
         self.crawler = crawler
+        
+        # Start the memory monitor task
+        memory_monitor = asyncio.create_task(self._memory_monitor_task())
+        
         if self.monitor:
             self.monitor.start()
-
+            
+        results = []
+        
         try:
-            active_tasks = []
-            task_queue = []
-            completed_count = 0
-            total_urls = len(urls)
-
             # Initialize task queue
             for url in urls:
                 task_id = str(uuid.uuid4())
                 if self.monitor:
                     self.monitor.add_task(task_id, url)
-                task_queue.append((url, task_id))
-
-            while completed_count < total_urls:
-                # Start new tasks if memory permits
-                while len(active_tasks) < self.max_session_permit and task_queue:
-                    if psutil.virtual_memory().percent >= self.memory_threshold_percent:
-                        await asyncio.sleep(self.check_interval)
-                        continue
-
-                    url, task_id = task_queue.pop(0)
-                    task = asyncio.create_task(self.crawl_url(url, config, task_id))
-                    active_tasks.append(task)
-
-                if not active_tasks and not task_queue:
-                    break
-
-                # Wait for any task to complete and yield results
+                # Add to queue with initial priority 0, retry count 0, and current time
+                await self.task_queue.put((0, (url, task_id, 0, time.time())))
+                
+            active_tasks = []
+            
+            # Process until both queues are empty
+            while not self.task_queue.empty() or active_tasks:
+                # If memory pressure is low, start new tasks
+                if not self.memory_pressure_mode and len(active_tasks) < self.max_session_permit:
+                    try:
+                        # Try to get a task with timeout to avoid blocking indefinitely
+                        priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for(
+                            self.task_queue.get(), timeout=0.1
+                        )
+                        
+                        # Create and start the task
+                        task = asyncio.create_task(
+                            self.crawl_url(url, config, task_id, retry_count)
+                        )
+                        active_tasks.append(task)
+                        
+                        # Update waiting time in monitor
+                        if self.monitor:
+                            wait_time = time.time() - enqueue_time
+                            self.monitor.update_task(
+                                task_id, 
+                                wait_time=wait_time,
+                                status=CrawlStatus.IN_PROGRESS
+                            )
+                            
+                    except asyncio.TimeoutError:
+                        # No tasks in queue, that's fine
+                        pass
+                        
+                # Wait for completion even if queue is starved
                 if active_tasks:
                     done, pending = await asyncio.wait(
-                        active_tasks,
-                        timeout=0.1,
-                        return_when=asyncio.FIRST_COMPLETED
+                        active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
                     )
+                    
+                    # Process completed tasks
                     for completed_task in done:
                         result = await completed_task
-                        completed_count += 1
-                        yield result
+                        results.append(result)
+                        
+                    # Update active tasks list
                     active_tasks = list(pending)
                 else:
-                    await asyncio.sleep(self.check_interval)
+                    # If no active tasks but still waiting, sleep briefly
+                    await asyncio.sleep(self.check_interval / 2)
+                    
+                # Update priorities for waiting tasks if needed
+                await self._update_queue_priorities()
+                
+            return results
 
+        except Exception as e:
+            if self.monitor:
+                self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")                
+        
         finally:
+            # Clean up
+            memory_monitor.cancel()
             if self.monitor:
                 self.monitor.stop()
+                
+    async def _update_queue_priorities(self):
+        """Periodically update priorities of items in the queue to prevent starvation"""
+        # Skip if queue is empty
+        if self.task_queue.empty():
+            return
+            
+        # Use a drain-and-refill approach to update all priorities
+        temp_items = []
+        
+        # Drain the queue (with a safety timeout to prevent blocking)
+        try:
+            drain_start = time.time()
+            while not self.task_queue.empty() and time.time() - drain_start < 5.0:  # 5 second safety timeout
+                try:
+                    # Get item from queue with timeout
+                    priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for(
+                        self.task_queue.get(), timeout=0.1
+                    )
+                    
+                    # Calculate new priority based on current wait time
+                    current_time = time.time()
+                    wait_time = current_time - enqueue_time
+                    new_priority = self._get_priority_score(wait_time, retry_count)
+                    
+                    # Store with updated priority
+                    temp_items.append((new_priority, (url, task_id, retry_count, enqueue_time)))
+                    
+                    # Update monitoring stats for this task
+                    if self.monitor and task_id in self.monitor.stats:
+                        self.monitor.update_task(task_id, wait_time=wait_time)
+                        
+                except asyncio.TimeoutError:
+                    # Queue might be empty or very slow
+                    break
+        except Exception as e:
+            # If anything goes wrong, make sure we refill the queue with what we've got
+            self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")
+        
+        # Calculate queue statistics
+        if temp_items and self.monitor:
+            total_queued = len(temp_items)
+            wait_times = [item[1][3] for item in temp_items]
+            highest_wait_time = time.time() - min(wait_times) if wait_times else 0
+            avg_wait_time = sum(time.time() - t for t in wait_times) / len(wait_times) if wait_times else 0
+            
+            # Update queue statistics in monitor
+            self.monitor.update_queue_statistics(
+                total_queued=total_queued,
+                highest_wait_time=highest_wait_time,
+                avg_wait_time=avg_wait_time
+            )
+        
+        # Sort by priority (lowest number = highest priority)
+        temp_items.sort(key=lambda x: x[0])
+        
+        # Refill the queue with updated priorities
+        for item in temp_items:
+            await self.task_queue.put(item)
+                
+    async def run_urls_stream(
+        self,
+        urls: List[str],
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlerTaskResult, None]:
+        self.crawler = crawler
+        
+        # Start the memory monitor task
+        memory_monitor = asyncio.create_task(self._memory_monitor_task())
+        
+        if self.monitor:
+            self.monitor.start()
+            
+        try:
+            # Initialize task queue
+            for url in urls:
+                task_id = str(uuid.uuid4())
+                if self.monitor:
+                    self.monitor.add_task(task_id, url)
+                # Add to queue with initial priority 0, retry count 0, and current time
+                await self.task_queue.put((0, (url, task_id, 0, time.time())))
+                
+            active_tasks = []
+            completed_count = 0
+            total_urls = len(urls)
+            
+            while completed_count < total_urls:
+                # If memory pressure is low, start new tasks
+                if not self.memory_pressure_mode and len(active_tasks) < self.max_session_permit:
+                    try:
+                        # Try to get a task with timeout
+                        priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for(
+                            self.task_queue.get(), timeout=0.1
+                        )
+                        
+                        # Create and start the task
+                        task = asyncio.create_task(
+                            self.crawl_url(url, config, task_id, retry_count)
+                        )
+                        active_tasks.append(task)
+                        
+                        # Update waiting time in monitor
+                        if self.monitor:
+                            wait_time = time.time() - enqueue_time
+                            self.monitor.update_task(
+                                task_id, 
+                                wait_time=wait_time,
+                                status=CrawlStatus.IN_PROGRESS
+                            )
+                            
+                    except asyncio.TimeoutError:
+                        # No tasks in queue, that's fine
+                        pass
+                        
+                # Process completed tasks and yield results
+                if active_tasks:
+                    done, pending = await asyncio.wait(
+                        active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
+                    )
+                    
+                    for completed_task in done:
+                        result = await completed_task
+                        
+                        # Only count as completed if it wasn't requeued
+                        if "requeued" not in result.error_message:
+                            completed_count += 1
+                            yield result
+                        
+                    # Update active tasks list
+                    active_tasks = list(pending)
+                else:
+                    # If no active tasks but still waiting, sleep briefly
+                    await asyncio.sleep(self.check_interval / 2)
+                
+                # Update priorities for waiting tasks if needed
+                await self._update_queue_priorities()
+                
+        finally:
+            # Clean up
+            memory_monitor.cancel()
+            if self.monitor:
+                self.monitor.stop()
+                
 
 class SemaphoreDispatcher(BaseDispatcher):
     def __init__(
@@ -544,7 +543,7 @@ class SemaphoreDispatcher(BaseDispatcher):
         task_id: str,
         semaphore: asyncio.Semaphore = None,
     ) -> CrawlerTaskResult:
-        start_time = datetime.now()
+        start_time = time.time()
         error_message = ""
         memory_usage = peak_memory = 0.0
 
@@ -577,7 +576,7 @@ class SemaphoreDispatcher(BaseDispatcher):
                             memory_usage=memory_usage,
                             peak_memory=peak_memory,
                             start_time=start_time,
-                            end_time=datetime.now(),
+                            end_time=time.time(),
                             error_message=error_message,
                         )
 
@@ -597,7 +596,7 @@ class SemaphoreDispatcher(BaseDispatcher):
             )
 
         finally:
-            end_time = datetime.now()
+            end_time = time.time()
             if self.monitor:
                 self.monitor.update_task(
                     task_id,
@@ -620,7 +619,7 @@ class SemaphoreDispatcher(BaseDispatcher):
 
     async def run_urls(
         self,
-        crawler: "AsyncWebCrawler",  # noqa: F821
+        crawler: AsyncWebCrawler,  # noqa: F821
         urls: List[str],
         config: CrawlerRunConfig,
     ) -> List[CrawlerTaskResult]:
@@ -644,4 +643,4 @@ class SemaphoreDispatcher(BaseDispatcher):
             return await asyncio.gather(*tasks, return_exceptions=True)
         finally:
             if self.monitor:
-                self.monitor.stop()
+                self.monitor.stop()

crawl4ai/async_dispatcher_.py

@@ -1,588 +0,0 @@
-from typing import Dict, Optional, List, Tuple
-from .async_configs import CrawlerRunConfig
-from .models import (
-    CrawlResult,
-    CrawlerTaskResult,
-    CrawlStatus,
-    DisplayMode,
-    CrawlStats,
-    DomainState,
-)
-
-from rich.live import Live
-from rich.table import Table
-from rich.console import Console
-from rich import box
-from datetime import datetime, timedelta
-
-import time
-import psutil
-import asyncio
-import uuid
-
-from urllib.parse import urlparse
-import random
-from abc import ABC, abstractmethod
-
-
-class RateLimiter:
-    def __init__(
-        self,
-        base_delay: Tuple[float, float] = (1.0, 3.0),
-        max_delay: float = 60.0,
-        max_retries: int = 3,
-        rate_limit_codes: List[int] = None,
-    ):
-        self.base_delay = base_delay
-        self.max_delay = max_delay
-        self.max_retries = max_retries
-        self.rate_limit_codes = rate_limit_codes or [429, 503]
-        self.domains: Dict[str, DomainState] = {}
-
-    def get_domain(self, url: str) -> str:
-        return urlparse(url).netloc
-
-    async def wait_if_needed(self, url: str) -> None:
-        domain = self.get_domain(url)
-        state = self.domains.get(domain)
-
-        if not state:
-            self.domains[domain] = DomainState()
-            state = self.domains[domain]
-
-        now = time.time()
-        if state.last_request_time:
-            wait_time = max(0, state.current_delay - (now - state.last_request_time))
-            if wait_time > 0:
-                await asyncio.sleep(wait_time)
-
-        # Random delay within base range if no current delay
-        if state.current_delay == 0:
-            state.current_delay = random.uniform(*self.base_delay)
-
-        state.last_request_time = time.time()
-
-    def update_delay(self, url: str, status_code: int) -> bool:
-        domain = self.get_domain(url)
-        state = self.domains[domain]
-
-        if status_code in self.rate_limit_codes:
-            state.fail_count += 1
-            if state.fail_count > self.max_retries:
-                return False
-
-            # Exponential backoff with random jitter
-            state.current_delay = min(
-                state.current_delay * 2 * random.uniform(0.75, 1.25), self.max_delay
-            )
-        else:
-            # Gradually reduce delay on success
-            state.current_delay = max(
-                random.uniform(*self.base_delay), state.current_delay * 0.75
-            )
-            state.fail_count = 0
-
-        return True
-
-
-class CrawlerMonitor:
-    def __init__(
-        self,
-        max_visible_rows: int = 15,
-        display_mode: DisplayMode = DisplayMode.DETAILED,
-    ):
-        self.console = Console()
-        self.max_visible_rows = max_visible_rows
-        self.display_mode = display_mode
-        self.stats: Dict[str, CrawlStats] = {}
-        self.process = psutil.Process()
-        self.start_time = datetime.now()
-        self.live = Live(self._create_table(), refresh_per_second=2)
-
-    def start(self):
-        self.live.start()
-
-    def stop(self):
-        self.live.stop()
-
-    def add_task(self, task_id: str, url: str):
-        self.stats[task_id] = CrawlStats(
-            task_id=task_id, url=url, status=CrawlStatus.QUEUED
-        )
-        self.live.update(self._create_table())
-
-    def update_task(self, task_id: str, **kwargs):
-        if task_id in self.stats:
-            for key, value in kwargs.items():
-                setattr(self.stats[task_id], key, value)
-            self.live.update(self._create_table())
-
-    def _create_aggregated_table(self) -> Table:
-        """Creates a compact table showing only aggregated statistics"""
-        table = Table(
-            box=box.ROUNDED,
-            title="Crawler Status Overview",
-            title_style="bold magenta",
-            header_style="bold blue",
-            show_lines=True,
-        )
-
-        # Calculate statistics
-        total_tasks = len(self.stats)
-        queued = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.QUEUED
-        )
-        in_progress = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.IN_PROGRESS
-        )
-        completed = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.COMPLETED
-        )
-        failed = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.FAILED
-        )
-
-        # Memory statistics
-        current_memory = self.process.memory_info().rss / (1024 * 1024)
-        total_task_memory = sum(stat.memory_usage for stat in self.stats.values())
-        peak_memory = max(
-            (stat.peak_memory for stat in self.stats.values()), default=0.0
-        )
-
-        # Duration
-        duration = datetime.now() - self.start_time
-
-        # Create status row
-        table.add_column("Status", style="bold cyan")
-        table.add_column("Count", justify="right")
-        table.add_column("Percentage", justify="right")
-
-        table.add_row("Total Tasks", str(total_tasks), "100%")
-        table.add_row(
-            "[yellow]In Queue[/yellow]",
-            str(queued),
-            f"{(queued/total_tasks*100):.1f}%" if total_tasks > 0 else "0%",
-        )
-        table.add_row(
-            "[blue]In Progress[/blue]",
-            str(in_progress),
-            f"{(in_progress/total_tasks*100):.1f}%" if total_tasks > 0 else "0%",
-        )
-        table.add_row(
-            "[green]Completed[/green]",
-            str(completed),
-            f"{(completed/total_tasks*100):.1f}%" if total_tasks > 0 else "0%",
-        )
-        table.add_row(
-            "[red]Failed[/red]",
-            str(failed),
-            f"{(failed/total_tasks*100):.1f}%" if total_tasks > 0 else "0%",
-        )
-
-        # Add memory information
-        table.add_section()
-        table.add_row(
-            "[magenta]Current Memory[/magenta]", f"{current_memory:.1f} MB", ""
-        )
-        table.add_row(
-            "[magenta]Total Task Memory[/magenta]", f"{total_task_memory:.1f} MB", ""
-        )
-        table.add_row(
-            "[magenta]Peak Task Memory[/magenta]", f"{peak_memory:.1f} MB", ""
-        )
-        table.add_row(
-            "[yellow]Runtime[/yellow]",
-            str(timedelta(seconds=int(duration.total_seconds()))),
-            "",
-        )
-
-        return table
-
-    def _create_detailed_table(self) -> Table:
-        table = Table(
-            box=box.ROUNDED,
-            title="Crawler Performance Monitor",
-            title_style="bold magenta",
-            header_style="bold blue",
-        )
-
-        # Add columns
-        table.add_column("Task ID", style="cyan", no_wrap=True)
-        table.add_column("URL", style="cyan", no_wrap=True)
-        table.add_column("Status", style="bold")
-        table.add_column("Memory (MB)", justify="right")
-        table.add_column("Peak (MB)", justify="right")
-        table.add_column("Duration", justify="right")
-        table.add_column("Info", style="italic")
-
-        # Add summary row
-        total_memory = sum(stat.memory_usage for stat in self.stats.values())
-        active_count = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.IN_PROGRESS
-        )
-        completed_count = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.COMPLETED
-        )
-        failed_count = sum(
-            1 for stat in self.stats.values() if stat.status == CrawlStatus.FAILED
-        )
-
-        table.add_row(
-            "[bold yellow]SUMMARY",
-            f"Total: {len(self.stats)}",
-            f"Active: {active_count}",
-            f"{total_memory:.1f}",
-            f"{self.process.memory_info().rss / (1024 * 1024):.1f}",
-            str(
-                timedelta(
-                    seconds=int((datetime.now() - self.start_time).total_seconds())
-                )
-            ),
-            f"✓{completed_count} ✗{failed_count}",
-            style="bold",
-        )
-
-        table.add_section()
-
-        # Add rows for each task
-        visible_stats = sorted(
-            self.stats.values(),
-            key=lambda x: (
-                x.status != CrawlStatus.IN_PROGRESS,
-                x.status != CrawlStatus.QUEUED,
-                x.end_time or datetime.max,
-            ),
-        )[: self.max_visible_rows]
-
-        for stat in visible_stats:
-            status_style = {
-                CrawlStatus.QUEUED: "white",
-                CrawlStatus.IN_PROGRESS: "yellow",
-                CrawlStatus.COMPLETED: "green",
-                CrawlStatus.FAILED: "red",
-            }[stat.status]
-
-            table.add_row(
-                stat.task_id[:8],  # Show first 8 chars of task ID
-                stat.url[:40] + "..." if len(stat.url) > 40 else stat.url,
-                f"[{status_style}]{stat.status.value}[/{status_style}]",
-                f"{stat.memory_usage:.1f}",
-                f"{stat.peak_memory:.1f}",
-                stat.duration,
-                stat.error_message[:40] if stat.error_message else "",
-            )
-
-        return table
-
-    def _create_table(self) -> Table:
-        """Creates the appropriate table based on display mode"""
-        if self.display_mode == DisplayMode.AGGREGATED:
-            return self._create_aggregated_table()
-        return self._create_detailed_table()
-
-
-class BaseDispatcher(ABC):
-    def __init__(
-        self,
-        rate_limiter: Optional[RateLimiter] = None,
-        monitor: Optional[CrawlerMonitor] = None,
-    ):
-        self.crawler = None
-        self._domain_last_hit: Dict[str, float] = {}
-        self.concurrent_sessions = 0
-        self.rate_limiter = rate_limiter
-        self.monitor = monitor
-
-    @abstractmethod
-    async def crawl_url(
-        self,
-        url: str,
-        config: CrawlerRunConfig,
-        task_id: str,
-        monitor: Optional[CrawlerMonitor] = None,
-    ) -> CrawlerTaskResult:
-        pass
-
-    @abstractmethod
-    async def run_urls(
-        self,
-        urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821
-        config: CrawlerRunConfig,
-        monitor: Optional[CrawlerMonitor] = None,
-    ) -> List[CrawlerTaskResult]:
-        pass
-
-
-class MemoryAdaptiveDispatcher(BaseDispatcher):
-    def __init__(
-        self,
-        memory_threshold_percent: float = 90.0,
-        check_interval: float = 1.0,
-        max_session_permit: int = 20,
-        memory_wait_timeout: float = 300.0,  # 5 minutes default timeout
-        rate_limiter: Optional[RateLimiter] = None,
-        monitor: Optional[CrawlerMonitor] = None,
-    ):
-        super().__init__(rate_limiter, monitor)
-        self.memory_threshold_percent = memory_threshold_percent
-        self.check_interval = check_interval
-        self.max_session_permit = max_session_permit
-        self.memory_wait_timeout = memory_wait_timeout
-
-    async def crawl_url(
-        self,
-        url: str,
-        config: CrawlerRunConfig,
-        task_id: str,
-    ) -> CrawlerTaskResult:
-        start_time = datetime.now()
-        error_message = ""
-        memory_usage = peak_memory = 0.0
-
-        try:
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id, status=CrawlStatus.IN_PROGRESS, start_time=start_time
-                )
-            self.concurrent_sessions += 1
-
-            if self.rate_limiter:
-                await self.rate_limiter.wait_if_needed(url)
-
-            process = psutil.Process()
-            start_memory = process.memory_info().rss / (1024 * 1024)
-            result = await self.crawler.arun(url, config=config, session_id=task_id)
-            end_memory = process.memory_info().rss / (1024 * 1024)
-
-            memory_usage = peak_memory = end_memory - start_memory
-
-            if self.rate_limiter and result.status_code:
-                if not self.rate_limiter.update_delay(url, result.status_code):
-                    error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
-                    if self.monitor:
-                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                    return CrawlerTaskResult(
-                        task_id=task_id,
-                        url=url,
-                        result=result,
-                        memory_usage=memory_usage,
-                        peak_memory=peak_memory,
-                        start_time=start_time,
-                        end_time=datetime.now(),
-                        error_message=error_message,
-                    )
-
-            if not result.success:
-                error_message = result.error_message
-                if self.monitor:
-                    self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-            elif self.monitor:
-                self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
-
-        except Exception as e:
-            error_message = str(e)
-            if self.monitor:
-                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-            result = CrawlResult(
-                url=url, html="", metadata={}, success=False, error_message=str(e)
-            )
-
-        finally:
-            end_time = datetime.now()
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id,
-                    end_time=end_time,
-                    memory_usage=memory_usage,
-                    peak_memory=peak_memory,
-                    error_message=error_message,
-                )
-            self.concurrent_sessions -= 1
-
-        return CrawlerTaskResult(
-            task_id=task_id,
-            url=url,
-            result=result,
-            memory_usage=memory_usage,
-            peak_memory=peak_memory,
-            start_time=start_time,
-            end_time=end_time,
-            error_message=error_message,
-        )
-
-    async def run_urls(
-        self,
-        urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821
-        config: CrawlerRunConfig,
-    ) -> List[CrawlerTaskResult]:
-        self.crawler = crawler
-
-        if self.monitor:
-            self.monitor.start()
-
-        try:
-            pending_tasks = []
-            active_tasks = []
-            task_queue = []
-
-            for url in urls:
-                task_id = str(uuid.uuid4())
-                if self.monitor:
-                    self.monitor.add_task(task_id, url)
-                task_queue.append((url, task_id))
-
-            while task_queue or active_tasks:
-                wait_start_time = time.time()
-                while len(active_tasks) < self.max_session_permit and task_queue:
-                    if psutil.virtual_memory().percent >= self.memory_threshold_percent:
-                        # Check if we've exceeded the timeout
-                        if time.time() - wait_start_time > self.memory_wait_timeout:
-                            raise MemoryError(
-                                f"Memory usage above threshold ({self.memory_threshold_percent}%) for more than {self.memory_wait_timeout} seconds"
-                            )
-                        await asyncio.sleep(self.check_interval)
-                        continue
-
-                    url, task_id = task_queue.pop(0)
-                    task = asyncio.create_task(self.crawl_url(url, config, task_id))
-                    active_tasks.append(task)
-
-                if not active_tasks:
-                    await asyncio.sleep(self.check_interval)
-                    continue
-
-                done, pending = await asyncio.wait(
-                    active_tasks, return_when=asyncio.FIRST_COMPLETED
-                )
-
-                pending_tasks.extend(done)
-                active_tasks = list(pending)
-
-            return await asyncio.gather(*pending_tasks)
-        finally:
-            if self.monitor:
-                self.monitor.stop()
-
-
-class SemaphoreDispatcher(BaseDispatcher):
-    def __init__(
-        self,
-        semaphore_count: int = 5,
-        max_session_permit: int = 20,
-        rate_limiter: Optional[RateLimiter] = None,
-        monitor: Optional[CrawlerMonitor] = None,
-    ):
-        super().__init__(rate_limiter, monitor)
-        self.semaphore_count = semaphore_count
-        self.max_session_permit = max_session_permit
-
-    async def crawl_url(
-        self,
-        url: str,
-        config: CrawlerRunConfig,
-        task_id: str,
-        semaphore: asyncio.Semaphore = None,
-    ) -> CrawlerTaskResult:
-        start_time = datetime.now()
-        error_message = ""
-        memory_usage = peak_memory = 0.0
-
-        try:
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id, status=CrawlStatus.IN_PROGRESS, start_time=start_time
-                )
-
-            if self.rate_limiter:
-                await self.rate_limiter.wait_if_needed(url)
-
-            async with semaphore:
-                process = psutil.Process()
-                start_memory = process.memory_info().rss / (1024 * 1024)
-                result = await self.crawler.arun(url, config=config, session_id=task_id)
-                end_memory = process.memory_info().rss / (1024 * 1024)
-
-                memory_usage = peak_memory = end_memory - start_memory
-
-                if self.rate_limiter and result.status_code:
-                    if not self.rate_limiter.update_delay(url, result.status_code):
-                        error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
-                        if self.monitor:
-                            self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                        return CrawlerTaskResult(
-                            task_id=task_id,
-                            url=url,
-                            result=result,
-                            memory_usage=memory_usage,
-                            peak_memory=peak_memory,
-                            start_time=start_time,
-                            end_time=datetime.now(),
-                            error_message=error_message,
-                        )
-
-                if not result.success:
-                    error_message = result.error_message
-                    if self.monitor:
-                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                elif self.monitor:
-                    self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
-
-        except Exception as e:
-            error_message = str(e)
-            if self.monitor:
-                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-            result = CrawlResult(
-                url=url, html="", metadata={}, success=False, error_message=str(e)
-            )
-
-        finally:
-            end_time = datetime.now()
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id,
-                    end_time=end_time,
-                    memory_usage=memory_usage,
-                    peak_memory=peak_memory,
-                    error_message=error_message,
-                )
-
-        return CrawlerTaskResult(
-            task_id=task_id,
-            url=url,
-            result=result,
-            memory_usage=memory_usage,
-            peak_memory=peak_memory,
-            start_time=start_time,
-            end_time=end_time,
-            error_message=error_message,
-        )
-
-    async def run_urls(
-        self,
-        crawler: "AsyncWebCrawler",  # noqa: F821
-        urls: List[str],
-        config: CrawlerRunConfig,
-    ) -> List[CrawlerTaskResult]:
-        self.crawler = crawler
-        if self.monitor:
-            self.monitor.start()
-
-        try:
-            semaphore = asyncio.Semaphore(self.semaphore_count)
-            tasks = []
-
-            for url in urls:
-                task_id = str(uuid.uuid4())
-                if self.monitor:
-                    self.monitor.add_task(task_id, url)
-                task = asyncio.create_task(
-                    self.crawl_url(url, config, task_id, semaphore)
-                )
-                tasks.append(task)
-
-            return await asyncio.gather(*tasks, return_exceptions=True)
-        finally:
-            if self.monitor:
-                self.monitor.stop()

crawl4ai/async_logger.py

@@ -1,3 +1,4 @@
+from abc import ABC, abstractmethod
 from enum import Enum
 from typing import Optional, Dict, Any
 from colorama import Fore, Style, init
@@ -13,7 +14,37 @@ class LogLevel(Enum):
     ERROR = 5
 
 
-class AsyncLogger:
+
+class AsyncLoggerBase(ABC):
+    @abstractmethod
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        pass
+
+    @abstractmethod
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        pass
+
+    @abstractmethod
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        pass
+
+    @abstractmethod
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        pass
+
+    @abstractmethod
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        pass
+
+    @abstractmethod
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+        pass
+
+    @abstractmethod
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+        pass
+
+class AsyncLogger(AsyncLoggerBase):
     """
     Asynchronous logger with support for colored console output and file logging.
     Supports templated messages with colored components.
@@ -125,9 +156,22 @@ class AsyncLogger:
                 formatted_message = message.format(**params)
 
                 # Then apply colors if specified
+                color_map = {
+                    "green": Fore.GREEN,
+                    "red": Fore.RED,
+                    "yellow": Fore.YELLOW,
+                    "blue": Fore.BLUE,
+                    "cyan": Fore.CYAN,
+                    "magenta": Fore.MAGENTA,
+                    "white": Fore.WHITE,
+                    "black": Fore.BLACK,
+                    "reset": Style.RESET_ALL,
+                }
                 if colors:
                     for key, color in colors.items():
                         # Find the formatted value in the message and wrap it with color
+                        if color in color_map:
+                            color = color_map[color]
                         if key in params:
                             value_str = str(params[key])
                             formatted_message = formatted_message.replace(
@@ -225,3 +269,55 @@ class AsyncLogger:
             tag=tag,
             params={"url": url, "url_length": url_length, "error": error},
         )
+
+class AsyncFileLogger(AsyncLoggerBase):
+    """
+    File-only asynchronous logger that writes logs to a specified file.
+    """
+
+    def __init__(self, log_file: str):
+        """
+        Initialize the file logger.
+
+        Args:
+            log_file: File path for logging
+        """
+        self.log_file = log_file
+        os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
+
+    def _write_to_file(self, level: str, message: str, tag: str):
+        """Write a message to the log file."""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+        with open(self.log_file, "a", encoding="utf-8") as f:
+            f.write(f"[{timestamp}] [{level}] [{tag}] {message}\n")
+
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        """Log a debug message to file."""
+        self._write_to_file("DEBUG", message, tag)
+
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        """Log an info message to file."""
+        self._write_to_file("INFO", message, tag)
+
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        """Log a success message to file."""
+        self._write_to_file("SUCCESS", message, tag)
+
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        """Log a warning message to file."""
+        self._write_to_file("WARNING", message, tag)
+
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        """Log an error message to file."""
+        self._write_to_file("ERROR", message, tag)
+
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+        """Log URL fetch status to file."""
+        status = "SUCCESS" if success else "FAILED"
+        message = f"{url[:url_length]}... | Status: {status} | Time: {timing:.2f}s"
+        self._write_to_file("URL_STATUS", message, tag)
+
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+        """Log error status to file."""
+        message = f"{url[:url_length]}... | Error: {error}"
+        self._write_to_file("ERROR", message, tag)

crawl4ai/async_webcrawler.py

@@ -1,7 +1,7 @@
+from .__version__ import __version__ as crawl4ai_version
 import os
 import sys
 import time
-import warnings
 from colorama import Fore
 from pathlib import Path
 from typing import Optional, List
@@ -10,30 +10,36 @@ import asyncio
 
 # from contextlib import nullcontext, asynccontextmanager
 from contextlib import asynccontextmanager
-from .models import CrawlResult, MarkdownGenerationResult, CrawlerTaskResult, DispatchResult
+from .models import (
+    CrawlResult,
+    MarkdownGenerationResult,
+    DispatchResult,
+    ScrapingResult,
+    CrawlResultContainer,
+    RunManyReturn
+)
 from .async_database import async_db_manager
 from .chunking_strategy import *  # noqa: F403
-from .chunking_strategy import RegexChunking, ChunkingStrategy, IdentityChunking
+from .chunking_strategy import IdentityChunking
 from .content_filter_strategy import *  # noqa: F403
-from .content_filter_strategy import RelevantContentFilter
-from .extraction_strategy import * # noqa: F403
-from .extraction_strategy import NoExtractionStrategy, ExtractionStrategy
+from .extraction_strategy import *  # noqa: F403
+from .extraction_strategy import NoExtractionStrategy
 from .async_crawler_strategy import (
     AsyncCrawlerStrategy,
     AsyncPlaywrightCrawlerStrategy,
     AsyncCrawlResponse,
 )
-from .cache_context import CacheMode, CacheContext, _legacy_to_cache_mode
+from .cache_context import CacheMode, CacheContext
 from .markdown_generation_strategy import (
     DefaultMarkdownGenerator,
     MarkdownGenerationStrategy,
 )
-from .async_logger import AsyncLogger
+from .deep_crawling import DeepCrawlDecorator
+from .async_logger import AsyncLogger, AsyncLoggerBase
 from .async_configs import BrowserConfig, CrawlerRunConfig
-from .async_dispatcher import * # noqa: F403
+from .async_dispatcher import *  # noqa: F403
 from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter
 
-from .config import MIN_WORD_THRESHOLD
 from .utils import (
     sanitize_input_encode,
     InvalidCSSSelectorError,
@@ -43,14 +49,6 @@ from .utils import (
     RobotsParser,
 )
 
-from typing import Union, AsyncGenerator, List, TypeVar
-from collections.abc import AsyncGenerator
-
-CrawlResultT = TypeVar('CrawlResultT', bound=CrawlResult)
-RunManyReturn = Union[List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
-
-from .__version__ import __version__ as crawl4ai_version
-
 
 class AsyncWebCrawler:
     """
@@ -76,31 +74,21 @@ class AsyncWebCrawler:
         await crawler.close()
         ```
 
-    Migration Guide:
-    Old way (deprecated):
-        crawler = AsyncWebCrawler(always_by_pass_cache=True, browser_type="chromium", headless=True)
-
-    New way (recommended):
-        browser_config = BrowserConfig(browser_type="chromium", headless=True)
-        crawler = AsyncWebCrawler(config=browser_config)
-
-
     Attributes:
         browser_config (BrowserConfig): Configuration object for browser settings.
         crawler_strategy (AsyncCrawlerStrategy): Strategy for crawling web pages.
         logger (AsyncLogger): Logger instance for recording events and errors.
-        always_bypass_cache (bool): Whether to always bypass cache.
         crawl4ai_folder (str): Directory for storing cache.
         base_directory (str): Base directory for storing cache.
         ready (bool): Whether the crawler is ready for use.
 
-        Methods:
-            start(): Start the crawler explicitly without using context manager.
-            close(): Close the crawler explicitly without using context manager.
-            arun(): Run the crawler for a single source: URL (web, local file, or raw HTML).
-            awarmup(): Perform warmup sequence.
-            arun_many(): Run the crawler for multiple sources.
-            aprocess_html(): Process HTML content.
+    Methods:
+        start(): Start the crawler explicitly without using context manager.
+        close(): Close the crawler explicitly without using context manager.
+        arun(): Run the crawler for a single source: URL (web, local file, or raw HTML).
+        awarmup(): Perform warmup sequence.
+        arun_many(): Run the crawler for multiple sources.
+        aprocess_html(): Process HTML content.
 
     Typical Usage:
         async with AsyncWebCrawler() as crawler:
@@ -121,81 +109,43 @@ class AsyncWebCrawler:
 
     def __init__(
         self,
-        crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
-        config: Optional[BrowserConfig] = None,
-        always_bypass_cache: bool = False,
-        always_by_pass_cache: Optional[bool] = None,  # Deprecated parameter
+        crawler_strategy: AsyncCrawlerStrategy = None,
+        config: BrowserConfig = None,
         base_directory: str = str(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())),
         thread_safe: bool = False,
+        logger: AsyncLoggerBase = None,
         **kwargs,
     ):
         """
         Initialize the AsyncWebCrawler.
 
         Args:
-            crawler_strategy: Strategy for crawling web pages. If None, will create AsyncPlaywrightCrawlerStrategy
-            config: Configuration object for browser settings. If None, will be created from kwargs
-            always_bypass_cache: Whether to always bypass cache (new parameter)
-            always_by_pass_cache: Deprecated, use always_bypass_cache instead
+            crawler_strategy: Strategy for crawling web pages. Default AsyncPlaywrightCrawlerStrategy
+            config: Configuration object for browser settings. Default BrowserConfig()
             base_directory: Base directory for storing cache
             thread_safe: Whether to use thread-safe operations
             **kwargs: Additional arguments for backwards compatibility
         """
         # Handle browser configuration
-        browser_config = config
-        if browser_config is not None:
-            if any(
-                k in kwargs
-                for k in [
-                    "browser_type",
-                    "headless",
-                    "viewport_width",
-                    "viewport_height",
-                ]
-            ):
-                self.logger.warning(
-                    message="Both browser_config and legacy browser parameters provided. browser_config will take precedence.",
-                    tag="WARNING",
-                )
-        else:
-            # Create browser config from kwargs for backwards compatibility
-            browser_config = BrowserConfig.from_kwargs(kwargs)
+        browser_config = config or BrowserConfig()
 
         self.browser_config = browser_config
 
         # Initialize logger first since other components may need it
-        self.logger = AsyncLogger(
+        self.logger = logger or AsyncLogger(
             log_file=os.path.join(base_directory, ".crawl4ai", "crawler.log"),
             verbose=self.browser_config.verbose,
             tag_width=10,
         )
 
         # Initialize crawler strategy
-        params = {k: v for k, v in kwargs.items() if k in ["browser_congig", "logger"]}
+        params = {k: v for k, v in kwargs.items() if k in ["browser_config", "logger"]}
         self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
             browser_config=browser_config,
             logger=self.logger,
             **params,  # Pass remaining kwargs for backwards compatibility
         )
 
-        # If craweler strategy doesnt have logger, use crawler logger
-        if not self.crawler_strategy.logger:
-            self.crawler_strategy.logger = self.logger
-
-        # Handle deprecated cache parameter
-        if always_by_pass_cache is not None:
-            if kwargs.get("warning", True):
-                warnings.warn(
-                    "'always_by_pass_cache' is deprecated and will be removed in version 0.5.0. "
-                    "Use 'always_bypass_cache' instead. "
-                    "Pass warning=False to suppress this warning.",
-                    DeprecationWarning,
-                    stacklevel=2,
-                )
-            self.always_bypass_cache = always_by_pass_cache
-        else:
-            self.always_bypass_cache = always_bypass_cache
-
         # Thread safety setup
         self._lock = asyncio.Lock() if thread_safe else None
 
@@ -209,21 +159,20 @@ class AsyncWebCrawler:
 
         self.ready = False
 
+        # Decorate arun method with deep crawling capabilities
+        self._deep_handler = DeepCrawlDecorator(self)
+        self.arun = self._deep_handler(self.arun)
+
     async def start(self):
         """
         Start the crawler explicitly without using context manager.
         This is equivalent to using 'async with' but gives more control over the lifecycle.
-
-        This method will:
-        1. Initialize the browser and context
-        2. Perform warmup sequence
-        3. Return the crawler instance for method chaining
-
         Returns:
             AsyncWebCrawler: The initialized crawler instance
         """
         await self.crawler_strategy.__aenter__()
-        await self.awarmup()
+        self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT")
+        self.ready = True
         return self
 
     async def close(self):
@@ -243,18 +192,6 @@ class AsyncWebCrawler:
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         await self.close()
 
-    async def awarmup(self):
-        """
-        Initialize the crawler with warm-up sequence.
-
-        This method:
-        1. Logs initialization info
-        2. Sets up browser configuration
-        3. Marks the crawler as ready
-        """
-        self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT")
-        self.ready = True
-
     @asynccontextmanager
     async def nullcontext(self):
         """异步空上下文管理器"""
@@ -263,26 +200,9 @@ class AsyncWebCrawler:
     async def arun(
         self,
         url: str,
-        config: Optional[CrawlerRunConfig] = None,
-        # Legacy parameters maintained for backwards compatibility
-        word_count_threshold=MIN_WORD_THRESHOLD,
-        extraction_strategy: ExtractionStrategy = None,
-        chunking_strategy: ChunkingStrategy = RegexChunking(),
-        content_filter: RelevantContentFilter = None,
-        cache_mode: Optional[CacheMode] = None,
-        # Deprecated cache parameters
-        bypass_cache: bool = False,
-        disable_cache: bool = False,
-        no_cache_read: bool = False,
-        no_cache_write: bool = False,
-        # Other legacy parameters
-        css_selector: str = None,
-        screenshot: bool = False,
-        pdf: bool = False,
-        user_agent: str = None,
-        verbose=True,
+        config: CrawlerRunConfig = None,
         **kwargs,
-    ) -> CrawlResult:
+    ) -> RunManyReturn:
         """
         Runs the crawler for a single source: URL (web, local file, or raw HTML).
 
@@ -311,70 +231,24 @@ class AsyncWebCrawler:
         Returns:
             CrawlResult: The result of crawling and processing
         """
-        crawler_config = config
+        # Auto-start if not ready
+        if not self.ready:
+            await self.start()
+
+        config = config or CrawlerRunConfig()
         if not isinstance(url, str) or not url:
             raise ValueError("Invalid URL, make sure the URL is a non-empty string")
 
         async with self._lock or self.nullcontext():
             try:
-                # Handle configuration
-                if crawler_config is not None:
-                    # if any(param is not None for param in [
-                    #     word_count_threshold, extraction_strategy, chunking_strategy,
-                    #     content_filter, cache_mode, css_selector, screenshot, pdf
-                    # ]):
-                    #     self.logger.warning(
-                    #         message="Both crawler_config and legacy parameters provided. crawler_config will take precedence.",
-                    #         tag="WARNING"
-                    #     )
-                    config = crawler_config
-                else:
-                    # Merge all parameters into a single kwargs dict for config creation
-                    config_kwargs = {
-                        "word_count_threshold": word_count_threshold,
-                        "extraction_strategy": extraction_strategy,
-                        "chunking_strategy": chunking_strategy,
-                        "content_filter": content_filter,
-                        "cache_mode": cache_mode,
-                        "bypass_cache": bypass_cache,
-                        "disable_cache": disable_cache,
-                        "no_cache_read": no_cache_read,
-                        "no_cache_write": no_cache_write,
-                        "css_selector": css_selector,
-                        "screenshot": screenshot,
-                        "pdf": pdf,
-                        "verbose": verbose,
-                        **kwargs,
-                    }
-                    config = CrawlerRunConfig.from_kwargs(config_kwargs)
-
-                # Handle deprecated cache parameters
-                if any([bypass_cache, disable_cache, no_cache_read, no_cache_write]):
-                    if kwargs.get("warning", True):
-                        warnings.warn(
-                            "Cache control boolean flags are deprecated and will be removed in version 0.5.0. "
-                            "Use 'cache_mode' parameter instead.",
-                            DeprecationWarning,
-                            stacklevel=2,
-                        )
-
-                    # Convert legacy parameters if cache_mode not provided
-                    if config.cache_mode is None:
-                        config.cache_mode = _legacy_to_cache_mode(
-                            disable_cache=disable_cache,
-                            bypass_cache=bypass_cache,
-                            no_cache_read=no_cache_read,
-                            no_cache_write=no_cache_write,
-                        )
+                self.logger.verbose = config.verbose
 
                 # Default to ENABLED if no cache mode specified
                 if config.cache_mode is None:
                     config.cache_mode = CacheMode.ENABLED
 
                 # Create cache context
-                cache_context = CacheContext(
-                    url, config.cache_mode, self.always_bypass_cache
-                )
+                cache_context = CacheContext(url, config.cache_mode, False)
 
                 # Initialize processing variables
                 async_response: AsyncCrawlResponse = None
@@ -401,7 +275,11 @@ class AsyncWebCrawler:
                     # If screenshot is requested but its not in cache, then set cache_result to None
                     screenshot_data = cached_result.screenshot
                     pdf_data = cached_result.pdf
-                    if config.screenshot and not screenshot or config.pdf and not pdf:
+                    # if config.screenshot and not screenshot or config.pdf and not pdf:
+                    if config.screenshot and not screenshot_data:
+                        cached_result = None
+
+                    if config.pdf and not pdf_data:
                         cached_result = None
 
                     self.logger.url_status(
@@ -411,26 +289,44 @@ class AsyncWebCrawler:
                         tag="FETCH",
                     )
 
+                # Update proxy configuration from rotation strategy if available
+                if config and config.proxy_rotation_strategy:
+                    next_proxy = await config.proxy_rotation_strategy.get_next_proxy()
+                    if next_proxy:
+                        self.logger.info(
+                            message="Switch proxy: {proxy}",
+                            tag="PROXY",
+                            params={"proxy": next_proxy.server},
+                        )
+                        config.proxy_config = next_proxy
+                        # config = config.clone(proxy_config=next_proxy)
+
                 # Fetch fresh content if needed
                 if not cached_result or not html:
                     t1 = time.perf_counter()
 
-                    if user_agent:
-                        self.crawler_strategy.update_user_agent(user_agent)
+                    if config.user_agent:
+                        self.crawler_strategy.update_user_agent(config.user_agent)
 
                     # Check robots.txt if enabled
                     if config and config.check_robots_txt:
-                        if not await self.robots_parser.can_fetch(url, self.browser_config.user_agent):
+                        if not await self.robots_parser.can_fetch(
+                            url, self.browser_config.user_agent
+                        ):
                             return CrawlResult(
                                 url=url,
                                 html="",
                                 success=False,
                                 status_code=403,
                                 error_message="Access denied by robots.txt",
-                                response_headers={"X-Robots-Status": "Blocked by robots.txt"}
+                                response_headers={
+                                    "X-Robots-Status": "Blocked by robots.txt"
+                                },
                             )
 
-                    # Pass config to crawl method
+                    ##############################
+                    # Call CrawlerStrategy.crawl #
+                    ##############################
                     async_response = await self.crawler_strategy.crawl(
                         url,
                         config=config,  # Pass the entire config object
@@ -439,6 +335,7 @@ class AsyncWebCrawler:
                     html = sanitize_input_encode(async_response.html)
                     screenshot_data = async_response.screenshot
                     pdf_data = async_response.pdf_data
+                    js_execution_result = async_response.js_execution_result
 
                     t2 = time.perf_counter()
                     self.logger.url_status(
@@ -448,8 +345,10 @@ class AsyncWebCrawler:
                         tag="FETCH",
                     )
 
-                    # Process the HTML content
-                    crawl_result : CrawlResult = await self.aprocess_html(
+                    ###############################################################
+                    # Process the HTML content, Call CrawlerStrategy.process_html #
+                    ###############################################################
+                    crawl_result: CrawlResult = await self.aprocess_html(
                         url=url,
                         html=html,
                         extracted_content=extracted_content,
@@ -465,30 +364,11 @@ class AsyncWebCrawler:
                     crawl_result.redirected_url = async_response.redirected_url or url
                     crawl_result.response_headers = async_response.response_headers
                     crawl_result.downloaded_files = async_response.downloaded_files
+                    crawl_result.js_execution_result = js_execution_result
                     crawl_result.ssl_certificate = (
                         async_response.ssl_certificate
                     )  # Add SSL certificate
 
-                    # # Check and set values from async_response to crawl_result
-                    # try:
-                    #     for key in vars(async_response):
-                    #         if hasattr(crawl_result, key):
-                    #             value = getattr(async_response, key, None)
-                    #             current_value = getattr(crawl_result, key, None)
-                    #             if value is not None and not current_value:
-                    #                 try:
-                    #                     setattr(crawl_result, key, value)
-                    #                 except Exception as e:
-                    #                     self.logger.warning(
-                    #                         message=f"Failed to set attribute {key}: {str(e)}",
-                    #                         tag="WARNING"
-                    #                     )
-                    # except Exception as e:
-                    #     self.logger.warning(
-                    #         message=f"Error copying response attributes: {str(e)}",
-                    #         tag="WARNING"
-                    #     )
-
                     crawl_result.success = bool(html)
                     crawl_result.session_id = getattr(config, "session_id", None)
 
@@ -510,7 +390,7 @@ class AsyncWebCrawler:
                     if cache_context.should_write() and not bool(cached_result):
                         await async_db_manager.acache_url(crawl_result)
 
-                    return crawl_result
+                    return CrawlResultContainer(crawl_result)
 
                 else:
                     self.logger.success(
@@ -527,7 +407,7 @@ class AsyncWebCrawler:
                     cached_result.success = bool(html)
                     cached_result.session_id = getattr(config, "session_id", None)
                     cached_result.redirected_url = cached_result.redirected_url or url
-                    return cached_result
+                    return CrawlResultContainer(cached_result)
 
             except Exception as e:
                 error_context = get_error_context(sys.exc_info())
@@ -538,8 +418,6 @@ class AsyncWebCrawler:
                     f"Error: {str(e)}\n\n"
                     f"Code context:\n{error_context['code_context']}"
                 )
-                # if not hasattr(e, "msg"):
-                #     e.msg = str(e)
 
                 self.logger.error_status(
                     url=url,
@@ -547,8 +425,10 @@ class AsyncWebCrawler:
                     tag="ERROR",
                 )
 
-                return CrawlResult(
-                    url=url, html="", success=False, error_message=error_message
+                return CrawlResultContainer(
+                    CrawlResult(
+                        url=url, html="", success=False, error_message=error_message
+                    )
                 )
 
     async def aprocess_html(
@@ -578,6 +458,7 @@ class AsyncWebCrawler:
         Returns:
             CrawlResult: Processed result containing extracted and formatted content
         """
+        cleaned_html = ""
         try:
             _url = url if not kwargs.get("is_raw_html", False) else "Raw HTML"
             t1 = time.perf_counter()
@@ -588,11 +469,15 @@ class AsyncWebCrawler:
                 scraping_strategy.logger = self.logger
 
             # Process HTML content
-            params = {k: v for k, v in config.to_dict().items() if k not in ["url"]}
+            params = config.__dict__.copy()
+            params.pop("url", None)
             # add keys from kwargs to params that doesn't exist in params
             params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
 
-            result = scraping_strategy.scrap(url, html, **params)
+            ################################
+            # Scraping Strategy Execution  #
+            ################################
+            result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
 
             if result is None:
                 raise ValueError(
@@ -618,7 +503,9 @@ class AsyncWebCrawler:
             links = result.links.model_dump()
             metadata = result.metadata
 
-        # Markdown Generation
+        ################################
+        # Generate Markdown            #
+        ################################
         markdown_generator: Optional[MarkdownGenerationStrategy] = (
             config.markdown_generator or DefaultMarkdownGenerator()
         )
@@ -634,24 +521,26 @@ class AsyncWebCrawler:
                 # html2text_options=kwargs.get('html2text', {})
             )
         )
-        markdown_v2 = markdown_result
-        markdown = sanitize_input_encode(markdown_result.raw_markdown)
 
         # Log processing completion
         self.logger.info(
-            message="Processed {url:.50}... | Time: {timing}ms",
+            message="{url:.50}... | Time: {timing}s",
             tag="SCRAPE",
-            params={"url": _url, "timing": int((time.perf_counter() - t1) * 1000)},
+            params={
+                "url": _url,
+                "timing": int((time.perf_counter() - t1) * 1000) / 1000,
+            },
         )
 
-        # Handle content extraction if needed
+        ################################
+        # Structured Content Extraction           #
+        ################################
         if (
             not bool(extracted_content)
             and config.extraction_strategy
             and not isinstance(config.extraction_strategy, NoExtractionStrategy)
         ):
             t1 = time.perf_counter()
-
             # Choose content based on input_format
             content_format = config.extraction_strategy.input_format
             if content_format == "fit_markdown" and not markdown_result.fit_markdown:
@@ -663,15 +552,16 @@ class AsyncWebCrawler:
                 content_format = "markdown"
 
             content = {
-                "markdown": markdown,
+                "markdown": markdown_result.raw_markdown,
                 "html": html,
-                "fit_markdown": markdown_result.raw_markdown,
-            }.get(content_format, markdown)
+                "cleaned_html": cleaned_html,
+                "fit_markdown": markdown_result.fit_markdown,
+            }.get(content_format, markdown_result.raw_markdown)
 
             # Use IdentityChunking for HTML input, otherwise use provided chunking strategy
             chunking = (
                 IdentityChunking()
-                if content_format == "html"
+                if content_format in ["html", "cleaned_html"]
                 else config.chunking_strategy
             )
             sections = chunking.chunk(content)
@@ -700,10 +590,7 @@ class AsyncWebCrawler:
             url=url,
             html=html,
             cleaned_html=cleaned_html,
-            markdown_v2=markdown_v2,
-            markdown=markdown,
-            fit_markdown=markdown_result.fit_markdown,
-            fit_html=markdown_result.fit_html,
+            markdown=markdown_result,
             media=media,
             links=links,
             metadata=metadata,
@@ -717,22 +604,22 @@ class AsyncWebCrawler:
     async def arun_many(
         self,
         urls: List[str],
-        config: Optional[CrawlerRunConfig] = None, 
+        config: Optional[CrawlerRunConfig] = None,
         dispatcher: Optional[BaseDispatcher] = None,
         # Legacy parameters maintained for backwards compatibility
-        word_count_threshold=MIN_WORD_THRESHOLD,
-        extraction_strategy: ExtractionStrategy = None,
-        chunking_strategy: ChunkingStrategy = RegexChunking(),
-        content_filter: RelevantContentFilter = None,
-        cache_mode: Optional[CacheMode] = None,
-        bypass_cache: bool = False,
-        css_selector: str = None,
-        screenshot: bool = False,
-        pdf: bool = False,
-        user_agent: str = None,
-        verbose=True,
-        **kwargs
-        ) -> RunManyReturn:
+        # word_count_threshold=MIN_WORD_THRESHOLD,
+        # extraction_strategy: ExtractionStrategy = None,
+        # chunking_strategy: ChunkingStrategy = RegexChunking(),
+        # content_filter: RelevantContentFilter = None,
+        # cache_mode: Optional[CacheMode] = None,
+        # bypass_cache: bool = False,
+        # css_selector: str = None,
+        # screenshot: bool = False,
+        # pdf: bool = False,
+        # user_agent: str = None,
+        # verbose=True,
+        **kwargs,
+    ) -> RunManyReturn:
         """
         Runs the crawler for multiple URLs concurrently using a configurable dispatcher strategy.
 
@@ -763,20 +650,21 @@ class AsyncWebCrawler:
         ):
             print(f"Processed {result.url}: {len(result.markdown)} chars")
         """
-        if config is None:
-            config = CrawlerRunConfig(
-                word_count_threshold=word_count_threshold,
-                extraction_strategy=extraction_strategy,
-                chunking_strategy=chunking_strategy,
-                content_filter=content_filter,
-                cache_mode=cache_mode,
-                bypass_cache=bypass_cache,
-                css_selector=css_selector,
-                screenshot=screenshot,
-                pdf=pdf,
-                verbose=verbose,
-                **kwargs,
-            )
+        config = config or CrawlerRunConfig()
+        # if config is None:
+        #     config = CrawlerRunConfig(
+        #         word_count_threshold=word_count_threshold,
+        #         extraction_strategy=extraction_strategy,
+        #         chunking_strategy=chunking_strategy,
+        #         content_filter=content_filter,
+        #         cache_mode=cache_mode,
+        #         bypass_cache=bypass_cache,
+        #         css_selector=css_selector,
+        #         screenshot=screenshot,
+        #         pdf=pdf,
+        #         verbose=verbose,
+        #         **kwargs,
+        #     )
 
         if dispatcher is None:
             dispatcher = MemoryAdaptiveDispatcher(
@@ -785,38 +673,34 @@ class AsyncWebCrawler:
                 ),
             )
 
-        transform_result = lambda task_result: (
-            setattr(task_result.result, 'dispatch_result', 
-                DispatchResult(
-                    task_id=task_result.task_id,
-                    memory_usage=task_result.memory_usage,
-                    peak_memory=task_result.peak_memory,
-                    start_time=task_result.start_time,
-                    end_time=task_result.end_time,
-                    error_message=task_result.error_message,
+        def transform_result(task_result):
+            return (
+                setattr(
+                    task_result.result,
+                    "dispatch_result",
+                    DispatchResult(
+                        task_id=task_result.task_id,
+                        memory_usage=task_result.memory_usage,
+                        peak_memory=task_result.peak_memory,
+                        start_time=task_result.start_time,
+                        end_time=task_result.end_time,
+                        error_message=task_result.error_message,
+                    ),
                 )
-            ) or task_result.result
-        )
+                or task_result.result
+            )
 
         stream = config.stream
-        
+
         if stream:
+
             async def result_transformer():
-                async for task_result in dispatcher.run_urls_stream(crawler=self, urls=urls, config=config):
+                async for task_result in dispatcher.run_urls_stream(
+                    crawler=self, urls=urls, config=config
+                ):
                     yield transform_result(task_result)
+
             return result_transformer()
         else:
             _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config)
-            return [transform_result(res) for res in _results]    
-
-    async def aclear_cache(self):
-        """Clear the cache database."""
-        await async_db_manager.cleanup()
-
-    async def aflush_cache(self):
-        """Flush the cache database."""
-        await async_db_manager.aflush_db()
-
-    async def aget_cache_size(self):
-        """Get the total number of cached items."""
-        return await async_db_manager.aget_total_count()
+            return [transform_result(res) for res in _results]

crawl4ai/browser/__init__.py

@@ -0,0 +1,23 @@
+"""Browser management module for Crawl4AI.
+
+This module provides browser management capabilities using different strategies
+for browser creation and interaction.
+"""
+
+from .manager import BrowserManager
+from .profiles import BrowserProfileManager
+from .models import DockerConfig
+from .docker_registry import DockerRegistry
+from .docker_utils import DockerUtils
+from .browser_hub import BrowserHub
+from .strategies import (
+    BaseBrowserStrategy,
+    PlaywrightBrowserStrategy,
+    CDPBrowserStrategy,
+    BuiltinBrowserStrategy,
+    DockerBrowserStrategy
+)
+
+__all__ = ['BrowserManager', 'BrowserProfileManager', 'DockerConfig', 'DockerRegistry', 'DockerUtils', 'BaseBrowserStrategy',
+           'PlaywrightBrowserStrategy', 'CDPBrowserStrategy', 'BuiltinBrowserStrategy',
+           'DockerBrowserStrategy', 'BrowserHub']

crawl4ai/browser/browser_hub.py

@@ -0,0 +1,184 @@
+# browser_hub_manager.py
+import hashlib
+import json
+import asyncio
+from typing import Dict, Optional, List, Tuple
+from .manager import BrowserManager, UnavailableBehavior
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+from ..async_logger import AsyncLogger
+
+class BrowserHub:
+    """
+    Manages Browser-Hub instances for sharing across multiple pipelines.
+    
+    This class provides centralized management for browser resources, allowing
+    multiple pipelines to share browser instances efficiently, connect to
+    existing browser hubs, or create new ones with custom configurations.
+    """
+    _instances: Dict[str, BrowserManager] = {}
+    _lock = asyncio.Lock()
+    
+    @classmethod
+    async def get_browser_manager(
+        cls, 
+        config: Optional[BrowserConfig] = None,
+        hub_id: Optional[str] = None,
+        connection_info: Optional[str] = None,
+        logger: Optional[AsyncLogger] = None,
+        max_browsers_per_config: int = 10,
+        max_pages_per_browser: int = 5,
+        initial_pool_size: int = 1,
+        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
+    ) -> BrowserManager:
+        """
+        Get an existing BrowserManager or create a new one based on parameters.
+        
+        Args:
+            config: Browser configuration for new hub
+            hub_id: Identifier for the hub instance
+            connection_info: Connection string for existing hub
+            logger: Logger for recording events and errors
+            max_browsers_per_config: Maximum browsers per configuration
+            max_pages_per_browser: Maximum pages per browser
+            initial_pool_size: Initial number of browsers to create
+            page_configs: Optional configurations for pre-warming pages
+            
+        Returns:
+            BrowserManager: The requested browser manager instance
+        """
+        async with cls._lock:
+            # Scenario 3: Use existing hub via connection info
+            if connection_info:
+                instance_key = f"connection:{connection_info}"
+                if instance_key not in cls._instances:
+                    cls._instances[instance_key] = await cls._connect_to_browser_hub(
+                        connection_info, logger
+                    )
+                return cls._instances[instance_key]
+                
+            # Scenario 2: Custom configured hub
+            if config:
+                config_hash = cls._hash_config(config)
+                instance_key = hub_id or f"config:{config_hash}"
+                if instance_key not in cls._instances:
+                    cls._instances[instance_key] = await cls._create_browser_manager(
+                        config, 
+                        logger,
+                        max_browsers_per_config,
+                        max_pages_per_browser,
+                        initial_pool_size,
+                        page_configs
+                    )
+                return cls._instances[instance_key]
+            
+            # Scenario 1: Default hub
+            instance_key = "default"
+            if instance_key not in cls._instances:
+                cls._instances[instance_key] = await cls._create_default_browser_hub(
+                    logger,
+                    max_browsers_per_config,
+                    max_pages_per_browser,
+                    initial_pool_size
+                )
+            return cls._instances[instance_key]
+    
+    @classmethod
+    async def _create_browser_manager(
+        cls, 
+        config: BrowserConfig,
+        logger: Optional[AsyncLogger],
+        max_browsers_per_config: int,
+        max_pages_per_browser: int,
+        initial_pool_size: int,
+        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
+    ) -> BrowserManager:
+        """Create a new browser hub with the specified configuration."""
+        manager = BrowserManager(
+            browser_config=config,
+            logger=logger,
+            unavailable_behavior=UnavailableBehavior.ON_DEMAND,
+            max_browsers_per_config=max_browsers_per_config,
+            max_pages_per_browser=max_pages_per_browser,
+        )
+        
+        # Initialize the pool
+        await manager.initialize_pool(
+            browser_configs=[config] if config else None,
+            browsers_per_config=initial_pool_size,
+            page_configs=page_configs
+        )
+        
+        return manager
+    
+    @classmethod
+    async def _create_default_browser_hub(
+        cls,
+        logger: Optional[AsyncLogger],
+        max_browsers_per_config: int,
+        max_pages_per_browser: int,
+        initial_pool_size: int
+    ) -> BrowserManager:
+        """Create a default browser hub with standard settings."""
+        config = BrowserConfig(headless=True)
+        return await cls._create_browser_manager(
+            config, 
+            logger, 
+            max_browsers_per_config, 
+            max_pages_per_browser, 
+            initial_pool_size,
+            None
+        )
+    
+    @classmethod
+    async def _connect_to_browser_hub(
+        cls, 
+        connection_info: str,
+        logger: Optional[AsyncLogger]
+    ) -> BrowserManager:
+        """
+        Connect to an existing browser hub.
+        
+        Note: This is a placeholder for future remote connection functionality.
+        Currently creates a local instance.
+        """
+        if logger:
+            logger.info(
+                message="Remote browser hub connections not yet implemented. Creating local instance.",
+                tag="BROWSER_HUB"
+            )
+        # For now, create a default local instance
+        return await cls._create_default_browser_hub(
+            logger, 
+            max_browsers_per_config=10, 
+            max_pages_per_browser=5, 
+            initial_pool_size=1
+        )
+    
+    @classmethod
+    def _hash_config(cls, config: BrowserConfig) -> str:
+        """Create a hash of the browser configuration for identification."""
+        # Convert config to dictionary, excluding any callable objects
+        config_dict = config.__dict__.copy()
+        for key in list(config_dict.keys()):
+            if callable(config_dict[key]):
+                del config_dict[key]
+        
+        # Convert to canonical JSON string
+        config_json = json.dumps(config_dict, sort_keys=True, default=str)
+        
+        # Hash the JSON
+        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
+        return config_hash
+    
+    @classmethod
+    async def shutdown_all(cls):
+        """Close all browser hub instances and clear the registry."""
+        async with cls._lock:
+            shutdown_tasks = []
+            for hub in cls._instances.values():
+                shutdown_tasks.append(hub.close())
+            
+            if shutdown_tasks:
+                await asyncio.gather(*shutdown_tasks)
+            
+            cls._instances.clear()

crawl4ai/browser/docker/alpine/connect.Dockerfile

@@ -0,0 +1,34 @@
+# ---------- Dockerfile ----------
+    FROM alpine:latest
+
+    # Combine everything in one RUN to keep layers minimal.
+    RUN apk update && apk upgrade && \
+        apk add --no-cache \
+            chromium \
+            nss \
+            freetype \
+            harfbuzz \
+            ca-certificates \
+            ttf-freefont \
+            socat \
+            curl && \
+        addgroup -S chromium && adduser -S chromium -G chromium && \
+        mkdir -p /data && chown chromium:chromium /data && \
+        rm -rf /var/cache/apk/*
+    
+    # Copy start script, then chown/chmod in one step
+    COPY start.sh /home/chromium/start.sh
+    RUN chown chromium:chromium /home/chromium/start.sh && \
+        chmod +x /home/chromium/start.sh
+    
+    USER chromium
+    WORKDIR /home/chromium
+    
+    # Expose port used by socat (mapping 9222→9223 or whichever you prefer)
+    EXPOSE 9223
+    
+    # Simple healthcheck: is the remote debug endpoint responding?
+    HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -f http://localhost:9222/json/version || exit 1
+    
+    CMD ["./start.sh"]
+    

crawl4ai/browser/docker/alpine/launch.Dockerfile

@@ -0,0 +1,27 @@
+# ---------- Dockerfile (Idle Version) ----------
+    FROM alpine:latest
+
+    # Install only Chromium and its dependencies in a single layer
+    RUN apk update && apk upgrade && \
+        apk add --no-cache \
+            chromium \
+            nss \
+            freetype \
+            harfbuzz \
+            ca-certificates \
+            ttf-freefont \
+            socat \
+            curl && \
+        addgroup -S chromium && adduser -S chromium -G chromium && \
+        mkdir -p /data && chown chromium:chromium /data && \
+        rm -rf /var/cache/apk/*
+    
+    ENV PATH="/usr/bin:/bin:/usr/sbin:/sbin"
+
+    # Switch to a non-root user for security
+    USER chromium
+    WORKDIR /home/chromium
+    
+    # Idle: container does nothing except stay alive
+    CMD ["tail", "-f", "/dev/null"]
+    

crawl4ai/browser/docker/debian/connect.Dockerfile

@@ -0,0 +1,23 @@
+# Use Debian 12 (Bookworm) slim for a small, stable base image
+FROM debian:bookworm-slim
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Chromium, socat, and basic fonts
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    chromium \
+    wget \
+    curl \
+    socat \
+    fonts-freefont-ttf \
+    fonts-noto-color-emoji && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Copy start.sh and make it executable
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+
+# Expose socat port (use host mapping, e.g. -p 9225:9223)
+EXPOSE 9223
+
+ENTRYPOINT ["/start.sh"]

crawl4ai/browser/docker_registry.py

@@ -0,0 +1,264 @@
+"""Docker registry module for Crawl4AI.
+
+This module provides a registry system for tracking and reusing Docker containers
+across browser sessions, improving performance and resource utilization.
+"""
+
+import os
+import json
+import time
+from typing import Dict, Optional
+
+from ..utils import get_home_folder
+
+
+class DockerRegistry:
+    """Manages a registry of Docker containers used for browser automation.
+    
+    This registry tracks containers by configuration hash, allowing reuse of appropriately
+    configured containers instead of creating new ones for each session.
+    
+    Attributes:
+        registry_file (str): Path to the registry file
+        containers (dict): Dictionary of container information
+        port_map (dict): Map of host ports to container IDs
+        last_port (int): Last port assigned
+    """
+    
+    def __init__(self, registry_file: Optional[str] = None):
+        """Initialize the registry with an optional path to the registry file.
+        
+        Args:
+            registry_file: Path to the registry file. If None, uses default path.
+        """
+        # Use the same file path as BuiltinBrowserStrategy by default
+        self.registry_file = registry_file or os.path.join(get_home_folder(), "builtin-browser", "browser_config.json")
+        self.containers = {}  # Still maintain this for backward compatibility
+        self.port_map = {}    # Will be populated from the shared file
+        self.last_port = 9222
+        self.load()
+    
+    def load(self):
+        """Load container registry from file."""
+        if os.path.exists(self.registry_file):
+            try:
+                with open(self.registry_file, 'r') as f:
+                    registry_data = json.load(f)
+                    
+                    # Initialize port_map if not present
+                    if "port_map" not in registry_data:
+                        registry_data["port_map"] = {}
+                    
+                    self.port_map = registry_data.get("port_map", {})
+                    
+                    # Extract container information from port_map entries of type "docker"
+                    self.containers = {}
+                    for port_str, browser_info in self.port_map.items():
+                        if browser_info.get("browser_type") == "docker" and "container_id" in browser_info:
+                            container_id = browser_info["container_id"]
+                            self.containers[container_id] = {
+                                "host_port": int(port_str),
+                                "config_hash": browser_info.get("config_hash", ""),
+                                "created_at": browser_info.get("created_at", time.time())
+                            }
+                    
+                    # Get last port if available
+                    if "last_port" in registry_data:
+                        self.last_port = registry_data["last_port"]
+                    else:
+                        # Find highest port in port_map
+                        ports = [int(p) for p in self.port_map.keys() if p.isdigit()]
+                        self.last_port = max(ports + [9222])
+                    
+            except Exception as e:
+                # Reset to defaults on error
+                print(f"Error loading registry: {e}")
+                self.containers = {}
+                self.port_map = {}
+                self.last_port = 9222
+        else:
+            # Initialize with defaults if file doesn't exist
+            self.containers = {}
+            self.port_map = {}
+            self.last_port = 9222
+    
+    def save(self):
+        """Save container registry to file."""
+        # First load the current file to avoid overwriting other browser types
+        current_data = {"port_map": {}, "last_port": self.last_port}
+        if os.path.exists(self.registry_file):
+            try:
+                with open(self.registry_file, 'r') as f:
+                    current_data = json.load(f)
+            except Exception:
+                pass
+        
+        # Create a new port_map dictionary
+        updated_port_map = {}
+        
+        # First, copy all non-docker entries from the existing port_map
+        for port_str, browser_info in current_data.get("port_map", {}).items():
+            if browser_info.get("browser_type") != "docker":
+                updated_port_map[port_str] = browser_info
+        
+        # Then add all current docker container entries
+        for container_id, container_info in self.containers.items():
+            port_str = str(container_info["host_port"])
+            updated_port_map[port_str] = {
+                "browser_type": "docker",
+                "container_id": container_id,
+                "cdp_url": f"http://localhost:{port_str}",
+                "config_hash": container_info["config_hash"],
+                "created_at": container_info["created_at"]
+            }
+        
+        # Replace the port_map with our updated version
+        current_data["port_map"] = updated_port_map
+        
+        # Update last_port
+        current_data["last_port"] = self.last_port
+        
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(self.registry_file), exist_ok=True)
+        
+        # Save the updated data
+        with open(self.registry_file, 'w') as f:
+            json.dump(current_data, f, indent=2)
+    
+    def register_container(self, container_id: str, host_port: int, config_hash: str, cdp_json_config: Optional[str] = None):
+        """Register a container with its configuration hash and port mapping.
+        
+        Args:
+            container_id: Docker container ID
+            host_port: Host port mapped to container
+            config_hash: Hash of configuration used to create container
+            cdp_json_config: CDP JSON configuration if available
+        """
+        self.containers[container_id] = {
+            "host_port": host_port,
+            "config_hash": config_hash,
+            "created_at": time.time()
+        }
+        
+        # Update port_map to maintain compatibility with BuiltinBrowserStrategy
+        port_str = str(host_port)
+        self.port_map[port_str] = {
+            "browser_type": "docker",
+            "container_id": container_id,
+            "cdp_url": f"http://localhost:{port_str}",
+            "config_hash": config_hash,
+            "created_at": time.time()
+        }
+        
+        if cdp_json_config:
+            self.port_map[port_str]["cdp_json_config"] = cdp_json_config
+        
+        self.save()
+    
+    def unregister_container(self, container_id: str):
+        """Unregister a container.
+        
+        Args:
+            container_id: Docker container ID to unregister
+        """
+        if container_id in self.containers:
+            host_port = self.containers[container_id]["host_port"]
+            port_str = str(host_port)
+            
+            # Remove from port_map
+            if port_str in self.port_map:
+                del self.port_map[port_str]
+                
+            # Remove from containers
+            del self.containers[container_id]
+            
+            self.save()
+    
+    async def find_container_by_config(self, config_hash: str, docker_utils) -> Optional[str]:
+        """Find a container that matches the given configuration hash.
+        
+        Args:
+            config_hash: Hash of configuration to match
+            docker_utils: DockerUtils instance to check running containers
+            
+        Returns:
+            Container ID if found, None otherwise
+        """
+        # Search through port_map for entries with matching config_hash
+        for port_str, browser_info in self.port_map.items():
+            if (browser_info.get("browser_type") == "docker" and 
+                browser_info.get("config_hash") == config_hash and 
+                "container_id" in browser_info):
+                
+                container_id = browser_info["container_id"]
+                if await docker_utils.is_container_running(container_id):
+                    return container_id
+        
+        return None
+    
+    def get_container_host_port(self, container_id: str) -> Optional[int]:
+        """Get the host port mapped to the container.
+        
+        Args:
+            container_id: Docker container ID
+            
+        Returns:
+            Host port if container is registered, None otherwise
+        """
+        if container_id in self.containers:
+            return self.containers[container_id]["host_port"]
+        return None
+    
+    def get_next_available_port(self, docker_utils) -> int:
+        """Get the next available host port for Docker mapping.
+        
+        Args:
+            docker_utils: DockerUtils instance to check port availability
+            
+        Returns:
+            Available port number
+        """
+        # Start from last port + 1
+        port = self.last_port + 1
+        
+        # Check if port is in use (either in our registry or system-wide)
+        while str(port) in self.port_map or docker_utils.is_port_in_use(port):
+            port += 1
+        
+        # Update last port
+        self.last_port = port
+        self.save()
+        
+        return port
+    
+    def get_container_config_hash(self, container_id: str) -> Optional[str]:
+        """Get the configuration hash for a container.
+        
+        Args:
+            container_id: Docker container ID
+            
+        Returns:
+            Configuration hash if container is registered, None otherwise
+        """
+        if container_id in self.containers:
+            return self.containers[container_id]["config_hash"]
+        return None
+    
+    def cleanup_stale_containers(self, docker_utils):
+        """Clean up containers that are no longer running.
+        
+        Args:
+            docker_utils: DockerUtils instance to check container status
+        """
+        to_remove = []
+        
+        # Find containers that are no longer running
+        for port_str, browser_info in self.port_map.items():
+            if browser_info.get("browser_type") == "docker" and "container_id" in browser_info:
+                container_id = browser_info["container_id"]
+                if not docker_utils.is_container_running(container_id):
+                    to_remove.append(container_id)
+        
+        # Remove stale containers
+        for container_id in to_remove:
+            self.unregister_container(container_id)

crawl4ai/browser/docker_utils.py

@@ -0,0 +1,661 @@
+import os
+import json
+import asyncio
+import hashlib
+import tempfile
+import shutil
+import socket
+import subprocess
+from typing import Dict, List, Optional, Tuple, Union
+
+
+class DockerUtils:
+    """Utility class for Docker operations in browser automation.
+
+    This class provides methods for managing Docker images, containers,
+    and related operations needed for browser automation. It handles
+    image building, container lifecycle, port management, and registry operations.
+
+    Attributes:
+        DOCKER_FOLDER (str): Path to folder containing Docker files
+        DOCKER_CONNECT_FILE (str): Path to Dockerfile for connect mode
+        DOCKER_LAUNCH_FILE (str): Path to Dockerfile for launch mode
+        DOCKER_START_SCRIPT (str): Path to startup script for connect mode
+        DEFAULT_CONNECT_IMAGE (str): Default image name for connect mode
+        DEFAULT_LAUNCH_IMAGE (str): Default image name for launch mode
+        logger: Optional logger instance
+    """
+
+    # File paths for Docker resources
+    DOCKER_FOLDER = os.path.join(os.path.dirname(__file__), "docker")
+    DOCKER_CONNECT_FILE = os.path.join(DOCKER_FOLDER, "connect.Dockerfile")
+    DOCKER_LAUNCH_FILE = os.path.join(DOCKER_FOLDER, "launch.Dockerfile")
+    DOCKER_START_SCRIPT = os.path.join(DOCKER_FOLDER, "start.sh")
+
+    # Default image names
+    DEFAULT_CONNECT_IMAGE = "crawl4ai/browser-connect:latest"
+    DEFAULT_LAUNCH_IMAGE = "crawl4ai/browser-launch:latest"
+
+    def __init__(self, logger=None):
+        """Initialize Docker utilities.
+
+        Args:
+            logger: Optional logger for recording operations
+        """
+        self.logger = logger
+
+    # Image Management Methods
+
+    async def check_image_exists(self, image_name: str) -> bool:
+        """Check if a Docker image exists.
+
+        Args:
+            image_name: Name of the Docker image to check
+
+        Returns:
+            bool: True if the image exists, False otherwise
+        """
+        cmd = ["docker", "image", "inspect", image_name]
+
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            _, _ = await process.communicate()
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.debug(
+                    f"Error checking if image exists: {str(e)}", tag="DOCKER"
+                )
+            return False
+
+    async def build_docker_image(
+        self,
+        image_name: str,
+        dockerfile_path: str,
+        files_to_copy: Dict[str, str] = None,
+    ) -> bool:
+        """Build a Docker image from a Dockerfile.
+
+        Args:
+            image_name: Name to give the built image
+            dockerfile_path: Path to the Dockerfile
+            files_to_copy: Dict of {dest_name: source_path} for files to copy to build context
+
+        Returns:
+            bool: True if image was built successfully, False otherwise
+        """
+        # Create a temporary build context
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Copy the Dockerfile
+            shutil.copy(dockerfile_path, os.path.join(temp_dir, "Dockerfile"))
+
+            # Copy any additional files needed
+            if files_to_copy:
+                for dest_name, source_path in files_to_copy.items():
+                    shutil.copy(source_path, os.path.join(temp_dir, dest_name))
+
+            # Build the image
+            cmd = ["docker", "build", "-t", image_name, temp_dir]
+
+            if self.logger:
+                self.logger.debug(
+                    f"Building Docker image with command: {' '.join(cmd)}", tag="DOCKER"
+                )
+
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            if process.returncode != 0:
+                if self.logger:
+                    self.logger.error(
+                        message="Failed to build Docker image: {error}",
+                        tag="DOCKER",
+                        params={"error": stderr.decode()},
+                    )
+                return False
+
+            if self.logger:
+                self.logger.success(
+                    f"Successfully built Docker image: {image_name}", tag="DOCKER"
+                )
+            return True
+
+    async def ensure_docker_image_exists(
+        self, image_name: str, mode: str = "connect"
+    ) -> str:
+        """Ensure the required Docker image exists, creating it if necessary.
+
+        Args:
+            image_name: Name of the Docker image
+            mode: Either "connect" or "launch" to determine which image to build
+
+        Returns:
+            str: Name of the available Docker image
+
+        Raises:
+            Exception: If image doesn't exist and can't be built
+        """
+        # If image name is not specified, use default based on mode
+        if not image_name:
+            image_name = (
+                self.DEFAULT_CONNECT_IMAGE
+                if mode == "connect"
+                else self.DEFAULT_LAUNCH_IMAGE
+            )
+
+        # Check if the image already exists
+        if await self.check_image_exists(image_name):
+            if self.logger:
+                self.logger.debug(
+                    f"Docker image {image_name} already exists", tag="DOCKER"
+                )
+            return image_name
+
+        # If we're using a custom image that doesn't exist, warn and fail
+        if (
+            image_name != self.DEFAULT_CONNECT_IMAGE
+            and image_name != self.DEFAULT_LAUNCH_IMAGE
+        ):
+            if self.logger:
+                self.logger.warning(
+                    f"Custom Docker image {image_name} not found and cannot be automatically created",
+                    tag="DOCKER",
+                )
+            raise Exception(f"Docker image {image_name} not found")
+
+        # Build the appropriate default image
+        if self.logger:
+            self.logger.info(
+                f"Docker image {image_name} not found, creating it now...", tag="DOCKER"
+            )
+
+        if mode == "connect":
+            success = await self.build_docker_image(
+                image_name,
+                self.DOCKER_CONNECT_FILE,
+                {"start.sh": self.DOCKER_START_SCRIPT},
+            )
+        else:
+            success = await self.build_docker_image(image_name, self.DOCKER_LAUNCH_FILE)
+
+        if not success:
+            raise Exception(f"Failed to create Docker image {image_name}")
+
+        return image_name
+
+    # Container Management Methods
+
+    async def create_container(
+        self,
+        image_name: str,
+        host_port: int,
+        container_name: Optional[str] = None,
+        volumes: List[str] = None,
+        network: Optional[str] = None,
+        env_vars: Dict[str, str] = None,
+        cpu_limit: float = 1.0,
+        memory_limit: str = "1.5g",
+        extra_args: List[str] = None,
+    ) -> Optional[str]:
+        """Create a new Docker container.
+
+        Args:
+            image_name: Docker image to use
+            host_port: Port on host to map to container port 9223
+            container_name: Optional name for the container
+            volumes: List of volume mappings (e.g., ["host_path:container_path"])
+            network: Optional Docker network to use
+            env_vars: Dictionary of environment variables
+            cpu_limit: CPU limit for the container
+            memory_limit: Memory limit for the container
+            extra_args: Additional docker run arguments
+
+        Returns:
+            str: Container ID if successful, None otherwise
+        """
+        # Prepare container command
+        cmd = [
+            "docker",
+            "run",
+            "--detach",
+        ]
+
+        # Add container name if specified
+        if container_name:
+            cmd.extend(["--name", container_name])
+
+        # Add port mapping
+        cmd.extend(["-p", f"{host_port}:9223"])
+
+        # Add volumes
+        if volumes:
+            for volume in volumes:
+                cmd.extend(["-v", volume])
+
+        # Add network if specified
+        if network:
+            cmd.extend(["--network", network])
+
+        # Add environment variables
+        if env_vars:
+            for key, value in env_vars.items():
+                cmd.extend(["-e", f"{key}={value}"])
+
+        # Add CPU and memory limits
+        if cpu_limit:
+            cmd.extend(["--cpus", str(cpu_limit)])
+        if memory_limit:
+            cmd.extend(["--memory", memory_limit])
+            cmd.extend(["--memory-swap", memory_limit])
+        if self.logger:
+            self.logger.debug(
+                f"Setting CPU limit: {cpu_limit}, Memory limit: {memory_limit}",
+                tag="DOCKER",
+            )
+
+        # Add extra args
+        if extra_args:
+            cmd.extend(extra_args)
+
+        # Add image
+        cmd.append(image_name)
+
+        if self.logger:
+            self.logger.debug(
+                f"Creating Docker container with command: {' '.join(cmd)}", tag="DOCKER"
+            )
+
+        # Run docker command
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            if process.returncode != 0:
+                if self.logger:
+                    self.logger.error(
+                        message="Failed to create Docker container: {error}",
+                        tag="DOCKER",
+                        params={"error": stderr.decode()},
+                    )
+                return None
+
+            # Get container ID
+            container_id = stdout.decode().strip()
+
+            if self.logger:
+                self.logger.success(
+                    f"Created Docker container: {container_id[:12]}", tag="DOCKER"
+                )
+
+            return container_id
+
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    message="Error creating Docker container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)},
+                )
+            return None
+
+    async def is_container_running(self, container_id: str) -> bool:
+        """Check if a container is running.
+
+        Args:
+            container_id: ID of the container to check
+
+        Returns:
+            bool: True if the container is running, False otherwise
+        """
+        cmd = ["docker", "inspect", "--format", "{{.State.Running}}", container_id]
+
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, _ = await process.communicate()
+
+            return process.returncode == 0 and stdout.decode().strip() == "true"
+        except Exception as e:
+            if self.logger:
+                self.logger.debug(
+                    f"Error checking if container is running: {str(e)}", tag="DOCKER"
+                )
+            return False
+
+    async def wait_for_container_ready(
+        self, container_id: str, timeout: int = 30
+    ) -> bool:
+        """Wait for the container to be in running state.
+
+        Args:
+            container_id: ID of the container to wait for
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            bool: True if container is ready, False if timeout occurred
+        """
+        for _ in range(timeout):
+            if await self.is_container_running(container_id):
+                return True
+            await asyncio.sleep(1)
+
+        if self.logger:
+            self.logger.warning(
+                f"Container {container_id[:12]} not ready after {timeout}s timeout",
+                tag="DOCKER",
+            )
+        return False
+
+    async def stop_container(self, container_id: str) -> bool:
+        """Stop a Docker container.
+
+        Args:
+            container_id: ID of the container to stop
+
+        Returns:
+            bool: True if stopped successfully, False otherwise
+        """
+        cmd = ["docker", "stop", container_id]
+
+        try:
+            process = await asyncio.create_subprocess_exec(*cmd)
+            await process.communicate()
+
+            if self.logger:
+                self.logger.debug(
+                    f"Stopped container: {container_id[:12]}", tag="DOCKER"
+                )
+
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to stop container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)},
+                )
+            return False
+
+    async def remove_container(self, container_id: str, force: bool = True) -> bool:
+        """Remove a Docker container.
+
+        Args:
+            container_id: ID of the container to remove
+            force: Whether to force removal
+
+        Returns:
+            bool: True if removed successfully, False otherwise
+        """
+        cmd = ["docker", "rm"]
+        if force:
+            cmd.append("-f")
+        cmd.append(container_id)
+
+        try:
+            process = await asyncio.create_subprocess_exec(*cmd)
+            await process.communicate()
+
+            if self.logger:
+                self.logger.debug(
+                    f"Removed container: {container_id[:12]}", tag="DOCKER"
+                )
+
+            return process.returncode == 0
+        except Exception as e:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to remove container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)},
+                )
+            return False
+
+    # Container Command Execution Methods
+
+    async def exec_in_container(
+        self, container_id: str, command: List[str], detach: bool = False
+    ) -> Tuple[int, str, str]:
+        """Execute a command in a running container.
+
+        Args:
+            container_id: ID of the container
+            command: Command to execute as a list of strings
+            detach: Whether to run the command in detached mode
+
+        Returns:
+            Tuple of (return_code, stdout, stderr)
+        """
+        cmd = ["docker", "exec"]
+        if detach:
+            cmd.append("-d")
+        cmd.append(container_id)
+        cmd.extend(command)
+
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+            )
+            stdout, stderr = await process.communicate()
+
+            return process.returncode, stdout.decode(), stderr.decode()
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    message="Error executing command in container: {error}",
+                    tag="DOCKER",
+                    params={"error": str(e)},
+                )
+            return -1, "", str(e)
+
+    async def start_socat_in_container(self, container_id: str) -> bool:
+        """Start socat in the container to map port 9222 to 9223.
+
+        Args:
+            container_id: ID of the container
+
+        Returns:
+            bool: True if socat started successfully, False otherwise
+        """
+        # Command to run socat as a background process
+        cmd = ["socat", "TCP-LISTEN:9223,fork", "TCP:localhost:9222"]
+
+        returncode, _, stderr = await self.exec_in_container(
+            container_id, cmd, detach=True
+        )
+
+        if returncode != 0:
+            if self.logger:
+                self.logger.error(
+                    message="Failed to start socat in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr},
+                )
+            return False
+
+        if self.logger:
+            self.logger.debug(
+                f"Started socat in container: {container_id[:12]}", tag="DOCKER"
+            )
+
+        # Wait a moment for socat to start
+        await asyncio.sleep(1)
+        return True
+
+    async def launch_chrome_in_container(
+        self, container_id: str, browser_args: List[str]
+    ) -> bool:
+        """Launch Chrome inside the container with specified arguments.
+
+        Args:
+            container_id: ID of the container
+            browser_args: Chrome command line arguments
+
+        Returns:
+            bool: True if Chrome started successfully, False otherwise
+        """
+        # Build Chrome command
+        chrome_cmd = ["chromium"]
+        chrome_cmd.extend(browser_args)
+
+        returncode, _, stderr = await self.exec_in_container(
+            container_id, chrome_cmd, detach=True
+        )
+
+        if returncode != 0:
+            if self.logger:
+                self.logger.error(
+                    message="Failed to launch Chrome in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr},
+                )
+            return False
+
+        if self.logger:
+            self.logger.debug(
+                f"Launched Chrome in container: {container_id[:12]}", tag="DOCKER"
+            )
+
+        return True
+
+    async def get_process_id_in_container(
+        self, container_id: str, process_name: str
+    ) -> Optional[int]:
+        """Get the process ID for a process in the container.
+
+        Args:
+            container_id: ID of the container
+            process_name: Name pattern to search for
+
+        Returns:
+            int: Process ID if found, None otherwise
+        """
+        cmd = ["pgrep", "-f", process_name]
+
+        returncode, stdout, _ = await self.exec_in_container(container_id, cmd)
+
+        if returncode == 0 and stdout.strip():
+            pid = int(stdout.strip().split("\n")[0])
+            return pid
+
+        return None
+
+    async def stop_process_in_container(self, container_id: str, pid: int) -> bool:
+        """Stop a process in the container by PID.
+
+        Args:
+            container_id: ID of the container
+            pid: Process ID to stop
+
+        Returns:
+            bool: True if process was stopped, False otherwise
+        """
+        cmd = ["kill", "-TERM", str(pid)]
+
+        returncode, _, stderr = await self.exec_in_container(container_id, cmd)
+
+        if returncode != 0:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to stop process in container: {error}",
+                    tag="DOCKER",
+                    params={"error": stderr},
+                )
+            return False
+
+        if self.logger:
+            self.logger.debug(
+                f"Stopped process {pid} in container: {container_id[:12]}", tag="DOCKER"
+            )
+
+        return True
+
+    # Network and Port Methods
+
+    async def wait_for_cdp_ready(self, host_port: int, timeout: int = 10) -> dict:
+        """Wait for the CDP endpoint to be ready.
+
+        Args:
+            host_port: Port to check for CDP endpoint
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            dict: CDP JSON config if ready, None if timeout occurred
+        """
+        import aiohttp
+
+        url = f"http://localhost:{host_port}/json/version"
+
+        for _ in range(timeout):
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(url, timeout=1) as response:
+                        if response.status == 200:
+                            if self.logger:
+                                self.logger.debug(
+                                    f"CDP endpoint ready on port {host_port}",
+                                    tag="DOCKER",
+                                )
+                            cdp_json_config = await response.json()
+                            if self.logger:
+                                self.logger.debug(
+                                    f"CDP JSON config: {cdp_json_config}", tag="DOCKER"
+                                )
+                            return cdp_json_config
+            except Exception:
+                pass
+            await asyncio.sleep(1)
+
+        if self.logger:
+            self.logger.warning(
+                f"CDP endpoint not ready on port {host_port} after {timeout}s timeout",
+                tag="DOCKER",
+            )
+        return None
+
+    def is_port_in_use(self, port: int) -> bool:
+        """Check if a port is already in use on the host.
+
+        Args:
+            port: Port number to check
+
+        Returns:
+            bool: True if port is in use, False otherwise
+        """
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            return s.connect_ex(("localhost", port)) == 0
+
+    def get_next_available_port(self, start_port: int = 9223) -> int:
+        """Get the next available port starting from a given port.
+
+        Args:
+            start_port: Port number to start checking from
+
+        Returns:
+            int: First available port number
+        """
+        port = start_port
+        while self.is_port_in_use(port):
+            port += 1
+        return port
+
+    # Configuration Hash Methods
+
+    def generate_config_hash(self, config_dict: Dict) -> str:
+        """Generate a hash of the configuration for container matching.
+
+        Args:
+            config_dict: Dictionary of configuration parameters
+
+        Returns:
+            str: Hash string uniquely identifying this configuration
+        """
+        # Convert to canonical JSON string and hash
+        config_json = json.dumps(config_dict, sort_keys=True)
+        return hashlib.sha256(config_json.encode()).hexdigest()

crawl4ai/browser/manager copy.py

@@ -0,0 +1,177 @@
+"""Browser manager module for Crawl4AI.
+
+This module provides a central browser management class that uses the
+strategy pattern internally while maintaining the existing API.
+It also implements a page pooling mechanism for improved performance.
+"""
+
+from typing import Optional, Tuple, List
+
+from playwright.async_api import Page, BrowserContext
+
+from ..async_logger import AsyncLogger
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+
+from .strategies import (
+    BaseBrowserStrategy,
+    PlaywrightBrowserStrategy,
+    CDPBrowserStrategy,
+    BuiltinBrowserStrategy,
+    DockerBrowserStrategy
+)
+
+class BrowserManager:
+    """Main interface for browser management in Crawl4AI.
+    
+    This class maintains backward compatibility with the existing implementation
+    while using the strategy pattern internally for different browser types.
+    
+    Attributes:
+        config (BrowserConfig): Configuration object containing all browser settings
+        logger: Logger instance for recording events and errors
+        browser: The browser instance
+        default_context: The default browser context
+        managed_browser: The managed browser instance
+        playwright: The Playwright instance
+        sessions: Dictionary to store session information
+        session_ttl: Session timeout in seconds
+    """
+    
+    def __init__(self, browser_config: Optional[BrowserConfig] = None, logger: Optional[AsyncLogger] = None):
+        """Initialize the BrowserManager with a browser configuration.
+        
+        Args:
+            browser_config: Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+        """
+        self.config = browser_config or BrowserConfig()
+        self.logger = logger
+        
+        # Create strategy based on configuration
+        self.strategy = self._create_strategy()
+        
+        # Initialize state variables for compatibility with existing code
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        
+        # For session management (from existing implementation)
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+    
+    def _create_strategy(self) -> BaseBrowserStrategy:
+        """Create appropriate browser strategy based on configuration.
+        
+        Returns:
+            BaseBrowserStrategy: The selected browser strategy
+        """
+        if self.config.browser_mode == "builtin":
+            return BuiltinBrowserStrategy(self.config, self.logger)
+        elif self.config.browser_mode == "docker":
+            if DockerBrowserStrategy is None:
+                if self.logger:
+                    self.logger.error(
+                        "Docker browser strategy requested but not available. "
+                        "Falling back to PlaywrightBrowserStrategy.",
+                        tag="BROWSER"
+                    )
+                return PlaywrightBrowserStrategy(self.config, self.logger)
+            return DockerBrowserStrategy(self.config, self.logger)
+        elif self.config.browser_mode == "cdp" or self.config.cdp_url or self.config.use_managed_browser:
+            return CDPBrowserStrategy(self.config, self.logger)
+        else:
+            return PlaywrightBrowserStrategy(self.config, self.logger)
+    
+    async def start(self):
+        """Start the browser instance and set up the default context.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Start the strategy
+        await self.strategy.start()
+        
+        # Update legacy references
+        self.browser = self.strategy.browser
+        self.default_context = self.strategy.default_context
+        
+        # Set browser process reference (for CDP strategy)
+        if hasattr(self.strategy, 'browser_process'):
+            self.managed_browser = self.strategy
+        
+        # Set Playwright reference
+        self.playwright = self.strategy.playwright
+        
+        # Sync sessions if needed
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+            self.session_ttl = self.strategy.session_ttl
+        
+        return self
+    
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page for the given configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Delegate to strategy
+        page, context = await self.strategy.get_page(crawlerRunConfig)
+        
+        # Sync sessions if needed
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+        
+        return page, context
+        
+    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
+        """Get multiple pages with the same configuration.
+        
+        This method efficiently creates multiple browser pages using the same configuration,
+        which is useful for parallel crawling of multiple URLs.
+        
+        Args:
+            crawlerRunConfig: Configuration for the pages
+            count: Number of pages to create
+            
+        Returns:
+            List of (Page, Context) tuples
+        """
+        # Delegate to strategy
+        pages = await self.strategy.get_pages(crawlerRunConfig, count)
+        
+        # Sync sessions if needed
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+            
+        return pages
+    
+    # Just for legacy compatibility
+    async def kill_session(self, session_id: str):
+        """Kill a browser session and clean up resources.
+        
+        Args:
+            session_id: The session ID to kill
+        """
+        # Handle kill_session via our strategy if it supports it
+        await self.strategy.kill_session(session_id)
+
+        # sync sessions if needed
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+    
+    async def close(self):
+        """Close the browser and clean up resources."""
+        # Delegate to strategy
+        await self.strategy.close()
+        
+        # Reset legacy references
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        self.sessions = {}

crawl4ai/browser/manager.py

@@ -0,0 +1,853 @@
+"""Browser manager module for Crawl4AI.
+
+This module provides a central browser management class that uses the
+strategy pattern internally while maintaining the existing API.
+It also implements browser pooling for improved performance.
+"""
+
+import asyncio
+import hashlib
+import json
+import math
+from enum import Enum
+from typing import Dict, List, Optional, Tuple, Any
+
+from playwright.async_api import Page, BrowserContext
+
+from ..async_logger import AsyncLogger
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+
+from .strategies import (
+    BaseBrowserStrategy,
+    PlaywrightBrowserStrategy,
+    CDPBrowserStrategy,
+    BuiltinBrowserStrategy,
+    DockerBrowserStrategy
+)
+
+class UnavailableBehavior(Enum):
+    """Behavior when no browser is available."""
+    ON_DEMAND = "on_demand"  # Create new browser on demand
+    PENDING = "pending"      # Wait until a browser is available
+    EXCEPTION = "exception"  # Raise an exception
+
+
+class BrowserManager:
+    """Main interface for browser management and pooling in Crawl4AI.
+    
+    This class maintains backward compatibility with the existing implementation
+    while using the strategy pattern internally for different browser types.
+    It also implements browser pooling for improved performance.
+    
+    Attributes:
+        config (BrowserConfig): Default configuration object for browsers
+        logger (AsyncLogger): Logger instance for recording events and errors
+        browser_pool (Dict): Dictionary to store browser instances by configuration
+        browser_in_use (Dict): Dictionary to track which browsers are in use
+        request_queues (Dict): Queues for pending requests by configuration
+        unavailable_behavior (UnavailableBehavior): Behavior when no browser is available
+    """
+    
+    def __init__(
+        self, 
+        browser_config: Optional[BrowserConfig] = None, 
+        logger: Optional[AsyncLogger] = None,
+        unavailable_behavior: UnavailableBehavior = UnavailableBehavior.EXCEPTION,
+        max_browsers_per_config: int = 10,
+        max_pages_per_browser: int = 5
+        ):
+        """Initialize the BrowserManager with a browser configuration.
+        
+        Args:
+            browser_config: Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+            unavailable_behavior: Behavior when no browser is available
+            max_browsers_per_config: Maximum number of browsers per configuration
+            max_pages_per_browser: Maximum number of pages per browser
+        """
+        self.config = browser_config or BrowserConfig()
+        self.logger = logger
+        self.unavailable_behavior = unavailable_behavior
+        self.max_browsers_per_config = max_browsers_per_config
+        self.max_pages_per_browser = max_pages_per_browser
+        
+        # Browser pool management
+        self.browser_pool = {}            # config_hash -> list of browser strategies
+        self.browser_in_use = {}          # strategy instance -> Boolean
+        self.request_queues = {}          # config_hash -> asyncio.Queue()
+        self._browser_locks = {}          # config_hash -> asyncio.Lock()
+        self._browser_pool_lock = asyncio.Lock()  # Global lock for pool modifications
+        
+        # Page pool management
+        self.page_pool = {}  # (browser_config_hash, crawler_config_hash) -> list of (page, context, strategy)
+        self._page_pool_lock = asyncio.Lock()
+            
+        self.browser_page_counts = {}  # strategy instance -> current page count
+        self._page_count_lock = asyncio.Lock()  # Lock for thread-safe access to page counts
+
+        # For session management (from existing implementation)
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+
+        # For legacy compatibility
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        self.strategy = None
+    
+    def _create_browser_config_hash(self, browser_config: BrowserConfig) -> str:
+        """Create a hash of the browser configuration for browser pooling.
+        
+        Args:
+            browser_config: Browser configuration
+            
+        Returns:
+            str: Hash of the browser configuration
+        """
+        # Convert config to dictionary, excluding any callable objects
+        config_dict = browser_config.__dict__.copy()
+        for key in list(config_dict.keys()):
+            if callable(config_dict[key]):
+                del config_dict[key]
+        
+        # Convert to canonical JSON string
+        config_json = json.dumps(config_dict, sort_keys=True, default=str)
+        
+        # Hash the JSON
+        config_hash = hashlib.sha256(config_json.encode()).hexdigest()
+        return config_hash
+    
+    def _create_strategy(self, browser_config: BrowserConfig) -> BaseBrowserStrategy:
+        """Create appropriate browser strategy based on configuration.
+        
+        Args:
+            browser_config: Browser configuration
+            
+        Returns:
+            BaseBrowserStrategy: The selected browser strategy
+        """
+        if browser_config.browser_mode == "builtin":
+            return BuiltinBrowserStrategy(browser_config, self.logger)
+        elif browser_config.browser_mode == "docker":
+            if DockerBrowserStrategy is None:
+                if self.logger:
+                    self.logger.error(
+                        "Docker browser strategy requested but not available. "
+                        "Falling back to PlaywrightBrowserStrategy.",
+                        tag="BROWSER"
+                    )
+                return PlaywrightBrowserStrategy(browser_config, self.logger)
+            return DockerBrowserStrategy(browser_config, self.logger)
+        elif browser_config.browser_mode == "cdp" or browser_config.cdp_url or browser_config.use_managed_browser:
+            return CDPBrowserStrategy(browser_config, self.logger)
+        else:
+            return PlaywrightBrowserStrategy(browser_config, self.logger)
+    
+    async def initialize_pool(
+        self, 
+        browser_configs: List[BrowserConfig] = None,
+        browsers_per_config: int = 1,
+        page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
+    ):
+        """Initialize the browser pool with multiple browser configurations.
+        
+        Args:
+            browser_configs: List of browser configurations to initialize
+            browsers_per_config: Number of browser instances per configuration
+            page_configs: Optional list of (browser_config, crawler_run_config, count) tuples
+                for pre-warming pages
+                
+        Returns:
+            self: For method chaining
+        """
+        if not browser_configs:
+            browser_configs = [self.config]
+        
+        # Calculate how many browsers we'll need based on page_configs
+        browsers_needed = {}
+        if page_configs:
+            for browser_config, _, page_count in page_configs:
+                config_hash = self._create_browser_config_hash(browser_config)
+                # Calculate browsers based on max_pages_per_browser
+                browsers_needed_for_config = math.ceil(page_count / self.max_pages_per_browser)
+                browsers_needed[config_hash] = max(
+                    browsers_needed.get(config_hash, 0),
+                    browsers_needed_for_config
+                )
+        
+        # Adjust browsers_per_config if needed to ensure enough capacity
+        config_browsers_needed = {}
+        for browser_config in browser_configs:
+            config_hash = self._create_browser_config_hash(browser_config)
+            
+            # Estimate browsers needed based on page requirements
+            browsers_for_config = browsers_per_config
+            if config_hash in browsers_needed:
+                browsers_for_config = max(browsers_for_config, browsers_needed[config_hash])
+            
+            config_browsers_needed[config_hash] = browsers_for_config
+            
+            # Update max_browsers_per_config if needed
+            if browsers_for_config > self.max_browsers_per_config:
+                self.max_browsers_per_config = browsers_for_config
+                if self.logger:
+                    self.logger.info(
+                        f"Increased max_browsers_per_config to {browsers_for_config} to accommodate page requirements",
+                        tag="POOL"
+                    )
+        
+        # Initialize locks and queues for each config
+        async with self._browser_pool_lock:
+            for browser_config in browser_configs:
+                config_hash = self._create_browser_config_hash(browser_config)
+                
+                # Initialize lock for this config if needed
+                if config_hash not in self._browser_locks:
+                    self._browser_locks[config_hash] = asyncio.Lock()
+                
+                # Initialize queue for this config if needed
+                if config_hash not in self.request_queues:
+                    self.request_queues[config_hash] = asyncio.Queue()
+                
+                # Initialize pool for this config if needed
+                if config_hash not in self.browser_pool:
+                    self.browser_pool[config_hash] = []
+        
+        # Create browser instances for each configuration in parallel
+        browser_tasks = []
+        
+        for browser_config in browser_configs:
+            config_hash = self._create_browser_config_hash(browser_config)
+            browsers_to_create = config_browsers_needed.get(
+                config_hash, 
+                browsers_per_config
+            ) - len(self.browser_pool.get(config_hash, []))
+            
+            if browsers_to_create <= 0:
+                continue
+                
+            for _ in range(browsers_to_create):
+                # Create a task for each browser initialization
+                task = self._create_and_add_browser(browser_config, config_hash)
+                browser_tasks.append(task)
+        
+        # Wait for all browser initializations to complete
+        if browser_tasks:
+            if self.logger:
+                self.logger.info(f"Initializing {len(browser_tasks)} browsers in parallel...", tag="POOL")
+            await asyncio.gather(*browser_tasks)
+        
+        # Pre-warm pages if requested
+        if page_configs:
+            page_tasks = []
+            for browser_config, crawler_run_config, count in page_configs:
+                task = self._prewarm_pages(browser_config, crawler_run_config, count)
+                page_tasks.append(task)
+            
+            if page_tasks:
+                if self.logger:
+                    self.logger.info(f"Pre-warming pages with {len(page_tasks)} configurations...", tag="POOL")
+                await asyncio.gather(*page_tasks)
+        
+        # Update legacy references
+        if self.browser_pool and next(iter(self.browser_pool.values()), []):
+            strategy = next(iter(self.browser_pool.values()))[0]
+            self.strategy = strategy
+            self.browser = strategy.browser
+            self.default_context = strategy.default_context
+            self.playwright = strategy.playwright
+        
+        return self
+
+    async def _create_and_add_browser(self, browser_config: BrowserConfig, config_hash: str):
+        """Create and add a browser to the pool.
+        
+        Args:
+            browser_config: Browser configuration
+            config_hash: Hash of the configuration
+        """
+        try:
+            strategy = self._create_strategy(browser_config)
+            await strategy.start()
+            
+            async with self._browser_pool_lock:
+                if config_hash not in self.browser_pool:
+                    self.browser_pool[config_hash] = []
+                self.browser_pool[config_hash].append(strategy)
+                self.browser_in_use[strategy] = False
+            
+            if self.logger:
+                self.logger.debug(
+                    f"Added browser to pool: {browser_config.browser_type} "
+                    f"({browser_config.browser_mode})", 
+                    tag="POOL"
+                )
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    f"Failed to create browser: {str(e)}", 
+                    tag="POOL"
+                )
+            raise
+
+    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
+        """Create a signature hash from crawler configuration.
+        
+        Args:
+            crawlerRunConfig: Crawler run configuration
+            
+        Returns:
+            str: Hash of the crawler configuration
+        """
+        config_dict = crawlerRunConfig.__dict__.copy()
+        # Exclude items that do not affect page creation
+        ephemeral_keys = [
+            "session_id",
+            "js_code",
+            "scraping_strategy",
+            "extraction_strategy",
+            "chunking_strategy",
+            "cache_mode",
+            "content_filter",
+            "semaphore_count",
+            "url"
+        ]
+        for key in ephemeral_keys:
+            if key in config_dict:
+                del config_dict[key]
+                
+        # Convert to canonical JSON string
+        config_json = json.dumps(config_dict, sort_keys=True, default=str)
+
+        # Hash the JSON
+        config_hash = hashlib.sha256(config_json.encode("utf-8")).hexdigest()
+        return config_hash            
+
+    async def _prewarm_pages(
+        self, 
+        browser_config: BrowserConfig, 
+        crawler_run_config: CrawlerRunConfig, 
+        count: int
+    ):
+        """Pre-warm pages for a specific configuration.
+        
+        Args:
+            browser_config: Browser configuration
+            crawler_run_config: Crawler run configuration
+            count: Number of pages to pre-warm
+        """
+        try:
+            # Create individual page tasks and run them in parallel
+            browser_config_hash = self._create_browser_config_hash(browser_config)
+            crawler_config_hash = self._make_config_signature(crawler_run_config)            
+            async def get_single_page():
+                strategy = await self.get_available_browser(browser_config)
+                try:
+                    page, context = await strategy.get_page(crawler_run_config)
+                    # Store config hashes on the page object for later retrieval
+                    setattr(page, "_browser_config_hash", browser_config_hash)
+                    setattr(page, "_crawler_config_hash", crawler_config_hash)                    
+                    return page, context, strategy
+                except Exception as e:
+                    # Release the browser back to the pool
+                    await self.release_browser(strategy, browser_config)
+                    raise e
+                
+            # Create tasks for parallel execution
+            page_tasks = [get_single_page() for _ in range(count)]
+            
+            # Execute all page creation tasks in parallel
+            pages_contexts_strategies = await asyncio.gather(*page_tasks)
+            
+            # Add pages to the page pool
+            browser_config_hash = self._create_browser_config_hash(browser_config)
+            crawler_config_hash = self._make_config_signature(crawler_run_config)
+            pool_key = (browser_config_hash, crawler_config_hash)
+            
+            async with self._page_pool_lock:
+                if pool_key not in self.page_pool:
+                    self.page_pool[pool_key] = []
+                
+                # Add all pages to the pool
+                self.page_pool[pool_key].extend(pages_contexts_strategies)
+                
+            if self.logger:
+                self.logger.debug(
+                    f"Pre-warmed {count} pages in parallel with config {crawler_run_config}", 
+                    tag="POOL"
+                )
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    f"Failed to pre-warm pages: {str(e)}", 
+                    tag="POOL"
+                )
+            raise
+    
+    async def get_available_browser(
+        self, 
+        browser_config: Optional[BrowserConfig] = None
+    ) -> BaseBrowserStrategy:
+        """Get an available browser from the pool for the given configuration.
+        
+        Args:
+            browser_config: Browser configuration to match
+            
+        Returns:
+            BaseBrowserStrategy: An available browser strategy
+            
+        Raises:
+            Exception: If no browser is available and behavior is EXCEPTION
+        """
+        browser_config = browser_config or self.config
+        config_hash = self._create_browser_config_hash(browser_config)
+        
+        async with self._browser_locks.get(config_hash, asyncio.Lock()):
+            # Check if we have browsers for this config
+            if config_hash not in self.browser_pool or not self.browser_pool[config_hash]:
+                if self.unavailable_behavior == UnavailableBehavior.ON_DEMAND:
+                    # Create a new browser on demand
+                    if self.logger:
+                        self.logger.info(
+                            f"1> Creating new browser on demand for config {config_hash[:8]}",
+                            tag="POOL"
+                        )
+                    
+                    # Initialize pool for this config if needed
+                    async with self._browser_pool_lock:
+                        if config_hash not in self.browser_pool:
+                            self.browser_pool[config_hash] = []
+                        
+                        strategy = self._create_strategy(browser_config)
+                        await strategy.start()
+
+                        self.browser_pool[config_hash].append(strategy)
+                        self.browser_in_use[strategy] = False
+
+                elif self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
+                    raise Exception(f"No browsers available for configuration {config_hash[:8]}")
+            
+            # Check for an available browser with capacity in the pool
+            for strategy in self.browser_pool[config_hash]:
+                # Check if this browser has capacity for more pages
+                async with self._page_count_lock:
+                    current_pages = self.browser_page_counts.get(strategy, 0)
+
+                    if current_pages < self.max_pages_per_browser:
+                        # Increment the page count
+                        self.browser_page_counts[strategy] = current_pages + 1
+
+                        self.browser_in_use[strategy] = True
+
+                        # Get browser information for better logging
+                        browser_type = getattr(strategy.config, 'browser_type', 'unknown')
+                        browser_mode = getattr(strategy.config, 'browser_mode', 'unknown')
+                        strategy_id = id(strategy)  # Use object ID as a unique identifier
+
+                        if self.logger:
+                            self.logger.debug(
+                                f"Selected browser #{strategy_id} ({browser_type}/{browser_mode}) - " 
+                                f"pages: {current_pages+1}/{self.max_pages_per_browser}",
+                                tag="POOL"
+                            )                        
+
+                        return strategy
+            
+            # All browsers are at capacity or in use
+            if self.unavailable_behavior == UnavailableBehavior.ON_DEMAND:
+                # Check if we've reached the maximum number of browsers
+                if len(self.browser_pool[config_hash]) >= self.max_browsers_per_config:
+                    if self.logger:
+                        self.logger.warning(
+                            f"Maximum browsers reached for config {config_hash[:8]} and all at page capacity",
+                            tag="POOL"
+                        )
+                    if self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
+                        raise Exception("Maximum browsers reached and all at page capacity")
+                
+                # Create a new browser on demand
+                if self.logger:
+                    self.logger.info(
+                        f"2> Creating new browser on demand for config {config_hash[:8]}",
+                        tag="POOL"
+                    )
+                
+                strategy = self._create_strategy(browser_config)
+                await strategy.start()
+                
+                async with self._browser_pool_lock:
+                    self.browser_pool[config_hash].append(strategy)
+                    self.browser_in_use[strategy] = True
+                
+                return strategy
+            
+            # If we get here, either behavior is EXCEPTION or PENDING
+            if self.unavailable_behavior == UnavailableBehavior.EXCEPTION:
+                raise Exception(f"All browsers in use or at page capacity for configuration {config_hash[:8]}")
+            
+            # For PENDING behavior, set up waiting mechanism
+            if config_hash not in self.request_queues:
+                self.request_queues[config_hash] = asyncio.Queue()
+            
+            # Create a future to wait on
+            future = asyncio.Future()
+            await self.request_queues[config_hash].put(future)
+            
+            if self.logger:
+                self.logger.debug(
+                    f"Waiting for available browser for config {config_hash[:8]}",
+                    tag="POOL"
+                )
+            
+            # Wait for a browser to become available
+            strategy = await future
+            return strategy
+
+    async def get_page(
+        self, 
+        crawlerRunConfig: CrawlerRunConfig,
+        browser_config: Optional[BrowserConfig] = None
+    ) -> Tuple[Page, BrowserContext, BaseBrowserStrategy]:
+        """Get a page from the browser pool."""
+        browser_config = browser_config or self.config
+        
+        # Check if we have a pre-warmed page available
+        browser_config_hash = self._create_browser_config_hash(browser_config)
+        crawler_config_hash = self._make_config_signature(crawlerRunConfig)
+        pool_key = (browser_config_hash, crawler_config_hash)
+        
+        # Try to get a page from the pool
+        async with self._page_pool_lock:
+            if pool_key in self.page_pool and self.page_pool[pool_key]:
+                # Get a page from the pool
+                page, context, strategy = self.page_pool[pool_key].pop()
+                
+                # Mark browser as in use (it already is, but ensure consistency)
+                self.browser_in_use[strategy] = True
+                
+                if self.logger:
+                    self.logger.debug(
+                        f"Using pre-warmed page for config {crawler_config_hash[:8]}", 
+                        tag="POOL"
+                    )
+                    
+                # Note: We don't increment page count since it was already counted when created
+                
+                return page, context, strategy
+        
+        # No pre-warmed page available, create a new one
+        # get_available_browser already increments the page count
+        strategy = await self.get_available_browser(browser_config)
+        
+        try:
+            # Get a page from the browser
+            page, context = await strategy.get_page(crawlerRunConfig)
+            
+            # Store config hashes on the page object for later retrieval
+            setattr(page, "_browser_config_hash", browser_config_hash)
+            setattr(page, "_crawler_config_hash", crawler_config_hash)
+            
+            return page, context, strategy
+        except Exception as e:
+            # Release the browser back to the pool and decrement the page count
+            await self.release_browser(strategy, browser_config, decrement_page_count=True)
+            raise e
+        
+    async def release_page(
+        self, 
+        page: Page, 
+        strategy: BaseBrowserStrategy, 
+        browser_config: Optional[BrowserConfig] = None,
+        keep_alive: bool = True,
+        return_to_pool: bool = True
+    ):
+        """Release a page back to the pool."""
+        browser_config = browser_config or self.config
+
+        page_url = page.url if page else None
+        
+        # If not keeping the page alive, close it and decrement count
+        if not keep_alive:
+            try:
+                await page.close()
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(
+                        f"Error closing page: {str(e)}",
+                        tag="POOL"
+                    )
+            # Release the browser with page count decrement
+            await self.release_browser(strategy, browser_config, decrement_page_count=True)
+            return
+        
+        # If returning to pool
+        if return_to_pool:
+            # Get the configuration hashes from the page object
+            browser_config_hash = getattr(page, "_browser_config_hash", None)
+            crawler_config_hash = getattr(page, "_crawler_config_hash", None)
+            
+            if browser_config_hash and crawler_config_hash:
+                pool_key = (browser_config_hash, crawler_config_hash)
+                
+                async with self._page_pool_lock:
+                    if pool_key not in self.page_pool:
+                        self.page_pool[pool_key] = []
+                    
+                    # Add page back to the pool
+                    self.page_pool[pool_key].append((page, page.context, strategy))
+                    
+                    if self.logger:
+                        self.logger.debug(
+                            f"Returned page to pool for config {crawler_config_hash[:8]}, url: {page_url}", 
+                            tag="POOL"
+                        )
+                    
+                    # Note: We don't decrement the page count here since the page is still "in use" 
+                    # from the browser's perspective, just in our pool
+                    return
+            else:
+                # If we can't identify the configuration, log a warning
+                if self.logger:
+                    self.logger.warning(
+                        "Cannot return page to pool - missing configuration hashes", 
+                        tag="POOL"
+                    )
+        
+        # If we got here, we couldn't return to pool, so just release the browser
+        await self.release_browser(strategy, browser_config, decrement_page_count=True)
+    
+    async def release_browser(
+        self, 
+        strategy: BaseBrowserStrategy, 
+        browser_config: Optional[BrowserConfig] = None,
+        decrement_page_count: bool = True
+    ):
+        """Release a browser back to the pool."""
+        browser_config = browser_config or self.config
+        config_hash = self._create_browser_config_hash(browser_config)
+        
+        # Decrement page count
+        if decrement_page_count:
+            async with self._page_count_lock:
+                current_count = self.browser_page_counts.get(strategy, 1)
+                self.browser_page_counts[strategy] = max(0, current_count - 1)
+                
+                if self.logger:
+                    self.logger.debug(
+                        f"Decremented page count for browser (now: {self.browser_page_counts[strategy]})",
+                        tag="POOL"
+                    )
+        
+        # Mark as not in use
+        self.browser_in_use[strategy] = False
+        
+        # Process any waiting requests
+        if config_hash in self.request_queues and not self.request_queues[config_hash].empty():
+            future = await self.request_queues[config_hash].get()
+            if not future.done():
+                future.set_result(strategy)
+
+    async def get_pages(
+        self, 
+        crawlerRunConfig: CrawlerRunConfig, 
+        count: int = 1,
+        browser_config: Optional[BrowserConfig] = None
+    ) -> List[Tuple[Page, BrowserContext, BaseBrowserStrategy]]:
+        """Get multiple pages from the browser pool.
+        
+        Args:
+            crawlerRunConfig: Configuration for the crawler run
+            count: Number of pages to get
+            browser_config: Browser configuration to use
+            
+        Returns:
+            List of (Page, Context, Strategy) tuples
+        """
+        results = []
+        for _ in range(count):
+            try:
+                result = await self.get_page(crawlerRunConfig, browser_config)
+                results.append(result)
+            except Exception as e:
+                # Release any pages we've already gotten
+                for page, _, strategy in results:
+                    await self.release_page(page, strategy, browser_config)
+                raise e
+        
+        return results
+
+    async def get_page_pool_status(self) -> Dict[str, Any]:
+        """Get information about the page pool status.
+        
+        Returns:
+            Dict with page pool status information
+        """
+        status = {
+            "total_pooled_pages": 0,
+            "configs": {}
+        }
+        
+        async with self._page_pool_lock:
+            for (browser_hash, crawler_hash), pages in self.page_pool.items():
+                config_key = f"{browser_hash[:8]}_{crawler_hash[:8]}"
+                status["configs"][config_key] = len(pages)
+                status["total_pooled_pages"] += len(pages)
+        
+        if self.logger:
+            self.logger.debug(
+                f"Page pool status: {status['total_pooled_pages']} pages available",
+                tag="POOL"
+            )
+        
+        return status
+
+    async def get_pool_status(self) -> Dict[str, Any]:
+        """Get information about the browser pool status.
+        
+        Returns:
+            Dict with pool status information
+        """
+        status = {
+            "total_browsers": 0,
+            "browsers_in_use": 0,
+            "total_pages": 0,
+            "configs": {}
+        }
+        
+        for config_hash, strategies in self.browser_pool.items():
+            config_pages = 0
+            in_use = 0
+            
+            for strategy in strategies:
+                is_in_use = self.browser_in_use.get(strategy, False)
+                if is_in_use:
+                    in_use += 1
+                
+                # Get page count for this browser
+                try:
+                    page_count = len(await strategy.get_opened_pages())
+                    config_pages += page_count
+                except Exception as e:
+                    if self.logger:
+                        self.logger.error(f"Error getting page count: {str(e)}", tag="POOL")
+            
+            config_status = {
+                "total_browsers": len(strategies),
+                "browsers_in_use": in_use,
+                "pages_open": config_pages,
+                "waiting_requests": self.request_queues.get(config_hash, asyncio.Queue()).qsize(),
+                "max_capacity": len(strategies) * self.max_pages_per_browser,
+                "utilization_pct": round((config_pages / (len(strategies) * self.max_pages_per_browser)) * 100, 1) 
+                    if strategies else 0
+            }
+            
+            status["configs"][config_hash] = config_status
+            status["total_browsers"] += config_status["total_browsers"]
+            status["browsers_in_use"] += config_status["browsers_in_use"]
+            status["total_pages"] += config_pages
+        
+        # Add overall utilization
+        if status["total_browsers"] > 0:
+            max_capacity = status["total_browsers"] * self.max_pages_per_browser
+            status["overall_utilization_pct"] = round((status["total_pages"] / max_capacity) * 100, 1)
+        else:
+            status["overall_utilization_pct"] = 0
+        
+        return status
+
+    async def start(self):
+        """Start at least one browser instance in the pool.
+        
+        This method is kept for backward compatibility.
+        
+        Returns:
+            self: For method chaining
+        """
+        await self.initialize_pool([self.config], 1)
+        return self
+        
+    async def kill_session(self, session_id: str):
+        """Kill a browser session and clean up resources.
+        
+        Delegated to the strategy. This method is kept for backward compatibility.
+        
+        Args:
+            session_id: The session ID to kill
+        """
+        if not self.strategy:
+            return
+            
+        await self.strategy.kill_session(session_id)
+        
+        # Sync sessions
+        if hasattr(self.strategy, 'sessions'):
+            self.sessions = self.strategy.sessions
+    
+    async def close(self):
+        """Close all browsers in the pool and clean up resources."""
+        # Close all browsers in the pool
+        for strategies in self.browser_pool.values():
+            for strategy in strategies:
+                try:
+                    await strategy.close()
+                except Exception as e:
+                    if self.logger:
+                        self.logger.error(
+                            f"Error closing browser: {str(e)}",
+                            tag="POOL"
+                        )
+        
+        # Clear pool data
+        self.browser_pool = {}
+        self.browser_in_use = {}
+        
+        # Reset legacy references
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+        self.strategy = None
+        self.sessions = {}
+
+
+async def create_browser_manager(
+    browser_config: Optional[BrowserConfig] = None,
+    logger: Optional[AsyncLogger] = None,
+    unavailable_behavior: UnavailableBehavior = UnavailableBehavior.EXCEPTION,
+    max_browsers_per_config: int = 10,
+    initial_pool_size: int = 1,
+    page_configs: Optional[List[Tuple[BrowserConfig, CrawlerRunConfig, int]]] = None
+) -> BrowserManager:
+    """Factory function to create and initialize a BrowserManager.
+    
+    Args:
+        browser_config: Configuration for the browsers
+        logger: Logger for recording events
+        unavailable_behavior: Behavior when no browser is available
+        max_browsers_per_config: Maximum browsers per configuration
+        initial_pool_size: Initial number of browsers per configuration
+        page_configs: Optional configurations for pre-warming pages
+        
+    Returns:
+        Initialized BrowserManager
+    """
+    manager = BrowserManager(
+        browser_config=browser_config,
+        logger=logger,
+        unavailable_behavior=unavailable_behavior,
+        max_browsers_per_config=max_browsers_per_config
+    )
+    
+    await manager.initialize_pool(
+        [browser_config] if browser_config else None,
+        initial_pool_size,
+        page_configs
+    )
+    
+    return manager
+
+
+
+
+

crawl4ai/browser/models.py

@@ -0,0 +1,143 @@
+"""Docker configuration module for Crawl4AI browser automation.
+
+This module provides configuration classes for Docker-based browser automation,
+allowing flexible configuration of Docker containers for browsing.
+"""
+
+from typing import Dict, List, Optional
+
+
+class DockerConfig:
+    """Configuration for Docker-based browser automation.
+    
+    This class contains Docker-specific settings to avoid cluttering BrowserConfig.
+    
+    Attributes:
+        mode (str): Docker operation mode - "connect" or "launch".
+            - "connect": Uses a container with Chrome already running
+            - "launch": Dynamically configures and starts Chrome in container
+        image (str): Docker image to use. If None, defaults from DockerUtils are used.
+        registry_file (str): Path to container registry file for persistence.
+        persistent (bool): Keep container running after browser closes.
+        remove_on_exit (bool): Remove container on exit when not persistent.
+        network (str): Docker network to use.
+        volumes (List[str]): Volume mappings (e.g., ["host_path:container_path"]).
+        env_vars (Dict[str, str]): Environment variables to set in container.
+        extra_args (List[str]): Additional docker run arguments.
+        host_port (int): Host port to map to container's 9223 port.
+        user_data_dir (str): Path to user data directory on host.
+        container_user_data_dir (str): Path to user data directory in container.
+    """
+    
+    def __init__(
+        self,
+        mode: str = "connect",                     # "connect" or "launch" 
+        image: Optional[str] = None,               # Docker image to use
+        registry_file: Optional[str] = None,       # Path to registry file
+        persistent: bool = False,                  # Keep container running after browser closes
+        remove_on_exit: bool = True,               # Remove container on exit when not persistent
+        network: Optional[str] = None,             # Docker network to use
+        volumes: List[str] = None,                 # Volume mappings
+        cpu_limit: float = 1.0,                    # CPU limit for the container
+        memory_limit: str = "1.5g",                # Memory limit for the container
+        env_vars: Dict[str, str] = None,           # Environment variables
+        host_port: Optional[int] = None,           # Host port to map to container's 9223
+        user_data_dir: Optional[str] = None,       # Path to user data directory on host
+        container_user_data_dir: str = "/data",    # Path to user data directory in container
+        extra_args: List[str] = None,              # Additional docker run arguments
+    ):
+        """Initialize Docker configuration.
+        
+        Args:
+            mode: Docker operation mode ("connect" or "launch")
+            image: Docker image to use
+            registry_file: Path to container registry file
+            persistent: Whether to keep container running after browser closes
+            remove_on_exit: Whether to remove container on exit when not persistent
+            network: Docker network to use
+            volumes: Volume mappings as list of strings
+            cpu_limit: CPU limit for the container
+            memory_limit: Memory limit for the container
+            env_vars: Environment variables as dictionary
+            extra_args: Additional docker run arguments
+            host_port: Host port to map to container's 9223
+            user_data_dir: Path to user data directory on host
+            container_user_data_dir: Path to user data directory in container
+        """
+        self.mode = mode
+        self.image = image  # If None, defaults will be used from DockerUtils
+        self.registry_file = registry_file
+        self.persistent = persistent
+        self.remove_on_exit = remove_on_exit
+        self.network = network
+        self.volumes = volumes or []
+        self.cpu_limit = cpu_limit
+        self.memory_limit = memory_limit
+        self.env_vars = env_vars or {}
+        self.extra_args = extra_args or []
+        self.host_port = host_port
+        self.user_data_dir = user_data_dir
+        self.container_user_data_dir = container_user_data_dir
+    
+    def to_dict(self) -> Dict:
+        """Convert this configuration to a dictionary.
+        
+        Returns:
+            Dictionary representation of this configuration
+        """
+        return {
+            "mode": self.mode,
+            "image": self.image,
+            "registry_file": self.registry_file,
+            "persistent": self.persistent,
+            "remove_on_exit": self.remove_on_exit,
+            "network": self.network,
+            "volumes": self.volumes,
+            "cpu_limit": self.cpu_limit,
+            "memory_limit": self.memory_limit,
+            "env_vars": self.env_vars,
+            "extra_args": self.extra_args,
+            "host_port": self.host_port,
+            "user_data_dir": self.user_data_dir,
+            "container_user_data_dir": self.container_user_data_dir
+        }
+        
+    @staticmethod
+    def from_kwargs(kwargs: Dict) -> "DockerConfig":
+        """Create a DockerConfig from a dictionary of keyword arguments.
+        
+        Args:
+            kwargs: Dictionary of configuration options
+            
+        Returns:
+            New DockerConfig instance
+        """
+        return DockerConfig(
+            mode=kwargs.get("mode", "connect"),
+            image=kwargs.get("image"),
+            registry_file=kwargs.get("registry_file"),
+            persistent=kwargs.get("persistent", False),
+            remove_on_exit=kwargs.get("remove_on_exit", True),
+            network=kwargs.get("network"),
+            volumes=kwargs.get("volumes"),
+            cpu_limit=kwargs.get("cpu_limit", 1.0),
+            memory_limit=kwargs.get("memory_limit", "1.5g"),
+            env_vars=kwargs.get("env_vars"),
+            extra_args=kwargs.get("extra_args"),
+            host_port=kwargs.get("host_port"),
+            user_data_dir=kwargs.get("user_data_dir"),
+            container_user_data_dir=kwargs.get("container_user_data_dir", "/data")
+        )
+        
+    def clone(self, **kwargs) -> "DockerConfig":
+        """Create a copy of this configuration with updated values.
+        
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+            
+        Returns:
+            DockerConfig: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return DockerConfig.from_kwargs(config_dict)

crawl4ai/browser/profiles.py

@@ -0,0 +1,457 @@
+"""Browser profile management module for Crawl4AI.
+
+This module provides functionality for creating and managing browser profiles
+that can be used for authenticated browsing.
+"""
+
+import os
+import asyncio
+import signal
+import sys
+import datetime
+import uuid
+import shutil
+from typing import List, Dict, Optional, Any
+from colorama import Fore, Style, init
+
+from ..async_configs import BrowserConfig
+from ..async_logger import AsyncLogger, AsyncLoggerBase
+from ..utils import get_home_folder
+
+class BrowserProfileManager:
+    """Manages browser profiles for Crawl4AI.
+    
+    This class provides functionality to create and manage browser profiles
+    that can be used for authenticated browsing with Crawl4AI.
+    
+    Profiles are stored by default in ~/.crawl4ai/profiles/
+    """
+    
+    def __init__(self, logger: Optional[AsyncLoggerBase] = None):
+        """Initialize the BrowserProfileManager.
+        
+        Args:
+            logger: Logger for outputting messages. If None, a default AsyncLogger is created.
+        """
+        # Initialize colorama for colorful terminal output
+        init()
+        
+        # Create a logger if not provided
+        if logger is None:
+            self.logger = AsyncLogger(verbose=True)
+        elif not isinstance(logger, AsyncLoggerBase):
+            self.logger = AsyncLogger(verbose=True)
+        else:
+            self.logger = logger
+            
+        # Ensure profiles directory exists
+        self.profiles_dir = os.path.join(get_home_folder(), "profiles")
+        os.makedirs(self.profiles_dir, exist_ok=True)
+    
+    async def create_profile(self, 
+                          profile_name: Optional[str] = None, 
+                          browser_config: Optional[BrowserConfig] = None) -> Optional[str]:
+        """Create a browser profile interactively.
+        
+        Args:
+            profile_name: Name for the profile. If None, a name is generated.
+            browser_config: Configuration for the browser. If None, a default configuration is used.
+                
+        Returns:
+            Path to the created profile directory, or None if creation failed
+        """
+        # Create default browser config if none provided
+        if browser_config is None:
+            browser_config = BrowserConfig(
+                browser_type="chromium",
+                headless=False,  # Must be visible for user interaction
+                verbose=True
+            )
+        else:
+            # Ensure headless is False for user interaction
+            browser_config.headless = False
+            
+        # Generate profile name if not provided
+        if not profile_name:
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"profile_{timestamp}_{uuid.uuid4().hex[:6]}"
+            
+        # Sanitize profile name (replace spaces and special chars)
+        profile_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in profile_name)
+        
+        # Set user data directory
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        os.makedirs(profile_path, exist_ok=True)
+        
+        # Print instructions for the user with colorama formatting
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="PROFILE")
+        self.logger.info(f"Creating browser profile: {Fore.GREEN}{profile_name}{Style.RESET_ALL}", tag="PROFILE")
+        self.logger.info(f"Profile directory: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+        
+        self.logger.info("\nInstructions:", tag="PROFILE")
+        self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE")
+        self.logger.info(f"2. {Fore.CYAN}Log in to websites{Style.RESET_ALL}, configure settings, etc. as needed.", tag="PROFILE")
+        self.logger.info(f"3. When you're done, {Fore.YELLOW}press 'q' in this terminal{Style.RESET_ALL} to close the browser.", tag="PROFILE")
+        self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE")
+        self.logger.info(f"{border}\n", tag="PROFILE")
+        
+        # Import the necessary classes with local imports to avoid circular references
+        from .strategies import CDPBrowserStrategy
+        
+        # Set browser config to use the profile path
+        browser_config.user_data_dir = profile_path
+        
+        # Create a CDP browser strategy for the profile creation
+        browser_strategy = CDPBrowserStrategy(browser_config, self.logger)
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="PROFILE")
+            await browser_strategy.close()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Profile creation interrupted. Profile may be incomplete.", tag="PROFILE")
+                sys.exit(1)
+                
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user is done with the browser
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' when you've finished using the browser...{Style.RESET_ALL}", tag="PROFILE")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser and saving profile...{Style.RESET_ALL}", tag="PROFILE")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if browser_strategy.browser_process and browser_strategy.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="PROFILE")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+        
+        try:
+            # Start the browser
+            await browser_strategy.start()
+            
+            # Check if browser started successfully
+            if not browser_strategy.browser_process:
+                self.logger.error("Failed to start browser process.", tag="PROFILE")
+                return None
+            
+            self.logger.info(f"Browser launched. {Fore.CYAN}Waiting for you to finish...{Style.RESET_ALL}", tag="PROFILE") 
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for either the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_strategy.browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_strategy.browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="PROFILE")
+                await browser_strategy.close()
+            
+            self.logger.success(f"Browser closed. Profile saved at: {Fore.GREEN}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+                
+        except Exception as e:
+            self.logger.error(f"Error creating profile: {str(e)}", tag="PROFILE")
+            await browser_strategy.close()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await browser_strategy.close()
+        
+        # Return the profile path
+        return profile_path
+    
+    def list_profiles(self) -> List[Dict[str, Any]]:
+        """List all available browser profiles.
+        
+        Returns:
+            List of dictionaries containing profile information
+        """
+        if not os.path.exists(self.profiles_dir):
+            return []
+            
+        profiles = []
+        
+        for name in os.listdir(self.profiles_dir):
+            profile_path = os.path.join(self.profiles_dir, name)
+            
+            # Skip if not a directory
+            if not os.path.isdir(profile_path):
+                continue
+                
+            # Check if this looks like a valid browser profile
+            # For Chromium: Look for Preferences file
+            # For Firefox: Look for prefs.js file
+            is_valid = False
+            
+            if os.path.exists(os.path.join(profile_path, "Preferences")) or \
+               os.path.exists(os.path.join(profile_path, "Default", "Preferences")):
+                is_valid = "chromium"
+            elif os.path.exists(os.path.join(profile_path, "prefs.js")):
+                is_valid = "firefox"
+                
+            if is_valid:
+                # Get creation time
+                created = datetime.datetime.fromtimestamp(
+                    os.path.getctime(profile_path)
+                )
+                
+                profiles.append({
+                    "name": name,
+                    "path": profile_path,
+                    "created": created,
+                    "type": is_valid
+                })
+                
+        # Sort by creation time, newest first
+        profiles.sort(key=lambda x: x["created"], reverse=True)
+        
+        return profiles
+    
+    def get_profile_path(self, profile_name: str) -> Optional[str]:
+        """Get the full path to a profile by name.
+        
+        Args:
+            profile_name: Name of the profile (not the full path)
+            
+        Returns:
+            Full path to the profile directory, or None if not found
+        """
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            # Check if profile_name itself is full path
+            if os.path.isabs(profile_name):
+                profile_path = profile_name
+            else:
+                return None
+        
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return None  # Not a valid browser profile
+            
+        return profile_path
+    
+    def delete_profile(self, profile_name_or_path: str) -> bool:
+        """Delete a browser profile by name or path.
+        
+        Args:
+            profile_name_or_path: Name of the profile or full path to profile directory
+            
+        Returns:
+            True if the profile was deleted successfully, False otherwise
+        """
+        # Determine if input is a name or a path
+        if os.path.isabs(profile_name_or_path):
+            # Full path provided
+            profile_path = profile_name_or_path
+        else:
+            # Just a name provided, construct path
+            profile_path = os.path.join(self.profiles_dir, profile_name_or_path)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            return False
+            
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return False  # Not a valid browser profile
+            
+        # Delete the profile directory
+        try:
+            shutil.rmtree(profile_path)
+            return True
+        except Exception:
+            return False
+            
+    async def interactive_manager(self, crawl_callback=None):
+        """Launch an interactive profile management console.
+        
+        Args:
+            crawl_callback: Function to call when selecting option to use 
+                a profile for crawling. It will be called with (profile_path, url).
+        """
+        while True:
+            self.logger.info(f"\n{Fore.CYAN}Profile Management Options:{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"1. {Fore.GREEN}Create a new profile{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"2. {Fore.YELLOW}List available profiles{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"3. {Fore.RED}Delete a profile{Style.RESET_ALL}", tag="MENU")
+            
+            # Only show crawl option if callback provided
+            if crawl_callback:
+                self.logger.info(f"4. {Fore.CYAN}Use a profile to crawl a website{Style.RESET_ALL}", tag="MENU")
+                self.logger.info(f"5. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "5"
+            else:
+                self.logger.info(f"4. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "4"
+            
+            choice = input(f"\n{Fore.CYAN}Enter your choice (1-{exit_option}): {Style.RESET_ALL}")
+            
+            if choice == "1":
+                # Create new profile
+                name = input(f"{Fore.GREEN}Enter a name for the new profile (or press Enter for auto-generated name): {Style.RESET_ALL}")
+                await self.create_profile(name or None)
+                
+            elif choice == "2":
+                # List profiles
+                profiles = self.list_profiles()
+                
+                if not profiles:
+                    self.logger.warning("  No profiles found. Create one first with option 1.", tag="PROFILES")
+                    continue
+                
+                # Print profile information with colorama formatting
+                self.logger.info("\nAvailable profiles:", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {Fore.CYAN}{profile['name']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Path: {Fore.YELLOW}{profile['path']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES")
+                    self.logger.info(f"    Browser type: {profile['type']}", tag="PROFILES")
+                    self.logger.info("", tag="PROFILES")  # Empty line for spacing
+                
+            elif choice == "3":
+                # Delete profile
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found to delete", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to delete
+                profile_idx = input(f"{Fore.RED}Enter the number of the profile to delete (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_name = profiles[idx]["name"]
+                        self.logger.info(f"Deleting profile: {Fore.YELLOW}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                        
+                        # Confirm deletion
+                        confirm = input(f"{Fore.RED}Are you sure you want to delete this profile? (y/n): {Style.RESET_ALL}")
+                        if confirm.lower() == 'y':
+                            success = self.delete_profile(profiles[idx]["path"])
+                            
+                            if success:
+                                self.logger.success(f"Profile {Fore.GREEN}{profile_name}{Style.RESET_ALL} deleted successfully", tag="PROFILES")
+                            else:
+                                self.logger.error(f"Failed to delete profile {Fore.RED}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == "4" and crawl_callback:
+                # Use profile to crawl a site
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found. Create one first.", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to use
+                profile_idx = input(f"{Fore.CYAN}Enter the number of the profile to use (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_path = profiles[idx]["path"]
+                        url = input(f"{Fore.CYAN}Enter the URL to crawl: {Style.RESET_ALL}")
+                        if url:
+                            # Call the provided crawl callback
+                            await crawl_callback(profile_path, url)
+                        else:
+                            self.logger.error("No URL provided", tag="CRAWL")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif (choice == "4" and not crawl_callback) or (choice == "5" and crawl_callback):
+                # Exit
+                self.logger.info("Exiting profile management", tag="MENU")
+                break
+                
+            else:
+                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")

crawl4ai/browser/strategies/__init__.py

@@ -0,0 +1,13 @@
+from .base import BaseBrowserStrategy
+from .cdp import CDPBrowserStrategy
+from .docker_strategy import DockerBrowserStrategy
+from .playwright import PlaywrightBrowserStrategy
+from .builtin import BuiltinBrowserStrategy
+
+__all__ = [
+    "BrowserStrategy",
+    "CDPBrowserStrategy",
+    "DockerBrowserStrategy",
+    "PlaywrightBrowserStrategy",
+    "BuiltinBrowserStrategy",
+]

crawl4ai/browser/strategies/base.py

@@ -0,0 +1,601 @@
+"""Browser strategies module for Crawl4AI.
+
+This module implements the browser strategy pattern for different
+browser implementations, including Playwright, CDP, and builtin browsers.
+"""
+
+from abc import ABC, abstractmethod
+import asyncio
+import json
+import hashlib
+import os
+import time
+from typing import Optional, Tuple, List
+
+from playwright.async_api import BrowserContext, Page
+
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig, CrawlerRunConfig
+from ...config import DOWNLOAD_PAGE_TIMEOUT
+from ...js_snippet import load_js_script
+from ..utils import get_playwright
+
+
+class BaseBrowserStrategy(ABC):
+    """Base class for all browser strategies.
+    
+    This abstract class defines the interface that all browser strategies
+    must implement. It handles common functionality like context caching,
+    browser configuration, and session management.
+    """
+    
+    _playwright_instance = None
+    
+    @classmethod
+    async def get_playwright(cls):
+        """Get or create a shared Playwright instance.
+        
+        Returns:
+            Playwright: The shared Playwright instance
+        """
+        # For now I dont want Singleton pattern for Playwright
+        if cls._playwright_instance is None or True:
+            cls._playwright_instance = await get_playwright()
+        return cls._playwright_instance
+        
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the strategy with configuration and logger.
+        
+        Args:
+            config: Browser configuration
+            logger: Logger for recording events and errors
+        """
+        self.config = config
+        self.logger = logger
+        self.browser = None
+        self.default_context = None
+        
+        # Context management
+        self.contexts_by_config = {}  # config_signature -> context
+
+        self._contexts_lock = asyncio.Lock()
+        
+        # Session management
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes default
+        
+        # Playwright instance
+        self.playwright = None
+    
+    @abstractmethod
+    async def start(self):
+        """Start the browser.
+        
+        This method should be implemented by concrete strategies to initialize 
+        the browser in the appropriate way (direct launch, CDP connection, etc.)
+        
+        Returns:
+            self: For method chaining
+        """
+        # Base implementation gets the playwright instance
+        self.playwright = await self.get_playwright()
+        return self
+    
+    @abstractmethod
+    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        pass
+
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page with specified configuration.
+        
+        This method should be implemented by concrete strategies to create
+        or retrieve a page according to their browser management approach.
+        
+        Args:
+            crawlerRunConfig: Crawler run configuration
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Clean up expired sessions first
+        self._cleanup_expired_sessions()
+
+        # If a session_id is provided and we already have it, reuse that page + context
+        if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions:
+            context, page, _ = self.sessions[crawlerRunConfig.session_id]
+            # Update last-used timestamp
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            return page, context
+        
+        page, context = await self._generate_page(crawlerRunConfig)
+
+        import uuid
+        setattr(page, "guid", uuid.uuid4())
+
+        # If a session_id is specified, store this session so we can reuse later
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            
+        return page, context        
+        pass
+    
+    async def get_pages(self, crawlerRunConfig: CrawlerRunConfig, count: int = 1) -> List[Tuple[Page, BrowserContext]]:
+        """Get multiple pages with the same configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration for the pages
+            count: Number of pages to create
+            
+        Returns:
+            List of (Page, Context) tuples
+        """
+        pages = []
+        for _ in range(count):
+            page, context = await self.get_page(crawlerRunConfig)
+            pages.append((page, context))
+        return pages
+       
+    async def get_opened_pages(self) -> List[Page]:
+        """Get all opened pages in the
+        browser.
+        """
+        return [page for context in self.contexts_by_config.values() for page in context.pages]
+
+    def _build_browser_args(self) -> dict:
+        """Build browser launch arguments from config.
+        
+        Returns:
+            dict: Browser launch arguments for Playwright
+        """
+        # Define common browser arguments that improve performance and stability
+        args = [
+            "--no-sandbox",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--window-position=0,0",
+            "--ignore-certificate-errors",
+            "--ignore-certificate-errors-spki-list",
+            "--window-position=400,0",
+            "--force-color-profile=srgb",
+            "--mute-audio",
+            "--disable-gpu",
+            "--disable-gpu-compositing",
+            "--disable-software-rasterizer",
+            "--disable-dev-shm-usage",
+            "--disable-infobars",
+            "--disable-blink-features=AutomationControlled",
+            "--disable-renderer-backgrounding",
+            "--disable-ipc-flooding-protection",
+            "--disable-background-timer-throttling",
+            f"--window-size={self.config.viewport_width},{self.config.viewport_height}",
+        ]
+
+        # Define browser disable options for light mode
+        browser_disable_options = [
+            "--disable-backgrounding-occluded-windows",
+            "--disable-breakpad",
+            "--disable-client-side-phishing-detection",
+            "--disable-component-extensions-with-background-pages",
+            "--disable-default-apps",
+            "--disable-extensions",
+            "--disable-features=TranslateUI",
+            "--disable-hang-monitor",
+            "--disable-popup-blocking",
+            "--disable-prompt-on-repost",
+            "--disable-sync",
+            "--metrics-recording-only",
+            "--password-store=basic",
+            "--use-mock-keychain",
+        ]
+
+        # Apply light mode settings if enabled
+        if self.config.light_mode:
+            args.extend(browser_disable_options)
+
+        # Apply text mode settings if enabled (disables images, JS, etc)
+        if self.config.text_mode:
+            args.extend([
+                "--blink-settings=imagesEnabled=false",
+                "--disable-remote-fonts",
+                "--disable-images",
+                "--disable-javascript",
+                "--disable-software-rasterizer",
+                "--disable-dev-shm-usage",
+            ])
+
+        # Add any extra arguments from the config
+        if self.config.extra_args:
+            args.extend(self.config.extra_args)
+
+        # Build the core browser args dictionary
+        browser_args = {"headless": self.config.headless, "args": args}
+
+        # Add chrome channel if specified
+        if self.config.chrome_channel:
+            browser_args["channel"] = self.config.chrome_channel
+
+        # Configure downloads
+        if self.config.accept_downloads:
+            browser_args["downloads_path"] = self.config.downloads_path or os.path.join(
+                os.getcwd(), "downloads"
+            )
+            os.makedirs(browser_args["downloads_path"], exist_ok=True)
+
+        # Check for user data directory
+        if self.config.user_data_dir:
+            # Ensure the directory exists
+            os.makedirs(self.config.user_data_dir, exist_ok=True)
+            browser_args["user_data_dir"] = self.config.user_data_dir
+        
+        # Configure proxy settings
+        if self.config.proxy or self.config.proxy_config:
+            from playwright.async_api import ProxySettings
+
+            proxy_settings = (
+                ProxySettings(server=self.config.proxy)
+                if self.config.proxy
+                else ProxySettings(
+                    server=self.config.proxy_config.server,
+                    username=self.config.proxy_config.username,
+                    password=self.config.proxy_config.password,
+                )
+            )
+            browser_args["proxy"] = proxy_settings
+
+        return browser_args
+    
+    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
+        """Create a signature hash from configuration for context caching.
+        
+        Converts the crawlerRunConfig into a dict, excludes ephemeral fields,
+        then returns a hash of the sorted JSON. This yields a stable signature
+        that identifies configurations requiring a unique browser context.
+        
+        Args:
+            crawlerRunConfig: Crawler run configuration
+            
+        Returns:
+            str: Unique hash for this configuration
+        """
+        config_dict = crawlerRunConfig.__dict__.copy()
+        # Exclude items that do not affect browser-level setup
+        ephemeral_keys = [
+            "session_id",
+            "js_code",
+            "scraping_strategy",
+            "extraction_strategy",
+            "chunking_strategy",
+            "cache_mode",
+            "content_filter",
+            "semaphore_count",
+            "url"
+        ]
+        for key in ephemeral_keys:
+            if key in config_dict:
+                del config_dict[key]
+                
+        # Convert to canonical JSON string
+        signature_json = json.dumps(config_dict, sort_keys=True, default=str)
+
+        # Hash the JSON so we get a compact, unique string
+        signature_hash = hashlib.sha256(signature_json.encode("utf-8")).hexdigest()
+        return signature_hash
+        
+    async def create_browser_context(self, crawlerRunConfig: Optional[CrawlerRunConfig] = None) -> BrowserContext:
+        """Creates and returns a new browser context with configured settings.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            BrowserContext: Browser context object with the specified configurations
+        """
+        if not self.browser:
+            raise ValueError("Browser must be initialized before creating context")
+            
+        # Base settings
+        user_agent = self.config.headers.get("User-Agent", self.config.user_agent) 
+        viewport_settings = {
+            "width": self.config.viewport_width,
+            "height": self.config.viewport_height,
+        }
+        proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
+        
+        # Define blocked extensions for resource optimization
+        blocked_extensions = [
+            # Images
+            "jpg", "jpeg", "png", "gif", "webp", "svg", "ico", "bmp", "tiff", "psd",
+            # Fonts
+            "woff", "woff2", "ttf", "otf", "eot",
+            # Media
+            "mp4", "webm", "ogg", "avi", "mov", "wmv", "flv", "m4v", "mp3", "wav", "aac",
+            "m4a", "opus", "flac",
+            # Documents
+            "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx",
+            # Archives
+            "zip", "rar", "7z", "tar", "gz",
+            # Scripts and data
+            "xml", "swf", "wasm",
+        ]
+
+        # Common context settings
+        context_settings = {
+            "user_agent": user_agent,
+            "viewport": viewport_settings,
+            "proxy": proxy_settings,
+            "accept_downloads": self.config.accept_downloads,
+            "storage_state": self.config.storage_state,
+            "ignore_https_errors": self.config.ignore_https_errors,
+            "device_scale_factor": 1.0,
+            "java_script_enabled": self.config.java_script_enabled,
+        }
+        
+        # Apply text mode settings if enabled
+        if self.config.text_mode:
+            text_mode_settings = {
+                "has_touch": False,
+                "is_mobile": False,
+                "java_script_enabled": False,  # Disable javascript in text mode
+            }
+            # Update context settings with text mode settings
+            context_settings.update(text_mode_settings)
+            if self.logger:
+                self.logger.debug("Text mode enabled for browser context", tag="BROWSER")
+        
+        # Handle storage state properly - this is key for persistence
+        if self.config.storage_state:
+            if self.logger:
+                if isinstance(self.config.storage_state, str):
+                    self.logger.debug(f"Using storage state from file: {self.config.storage_state}", tag="BROWSER")
+                else:
+                    self.logger.debug("Using storage state from config object", tag="BROWSER")
+
+        if self.config.user_data_dir:
+            # For CDP-based browsers, storage persistence is typically handled by the user_data_dir
+            # at the browser level, but we'll create a storage_state location for Playwright as well
+            storage_path = os.path.join(self.config.user_data_dir, "storage_state.json")
+            if not os.path.exists(storage_path):
+                # Create parent directory if it doesn't exist
+                os.makedirs(os.path.dirname(storage_path), exist_ok=True)
+                with open(storage_path, "w") as f:
+                    json.dump({}, f)
+            self.config.storage_state = storage_path
+
+            if self.logger:
+                self.logger.debug(f"Using user data directory: {self.config.user_data_dir}", tag="BROWSER")
+        
+        # Apply crawler-specific configurations if provided
+        if crawlerRunConfig:
+            # Check if there is value for crawlerRunConfig.proxy_config set add that to context
+            if crawlerRunConfig.proxy_config:
+                proxy_settings = {
+                    "server": crawlerRunConfig.proxy_config.server,
+                }
+                if crawlerRunConfig.proxy_config.username:
+                    proxy_settings.update({
+                        "username": crawlerRunConfig.proxy_config.username,
+                        "password": crawlerRunConfig.proxy_config.password,
+                    })
+                context_settings["proxy"] = proxy_settings
+                
+        # Create and return the context
+        try:
+            # Create the context with appropriate settings
+            context = await self.browser.new_context(**context_settings)
+            
+            # Apply text mode resource blocking if enabled
+            if self.config.text_mode:
+                # Create and apply route patterns for each extension
+                for ext in blocked_extensions:
+                    await context.route(f"**/*.{ext}", lambda route: route.abort())
+                    
+            return context
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error creating browser context: {str(e)}", tag="BROWSER")
+            # Fallback to basic context creation if the advanced settings fail
+            return await self.browser.new_context()
+        
+    async def setup_context(self, context: BrowserContext, crawlerRunConfig: Optional[CrawlerRunConfig] = None):
+        """Set up a browser context with the configured options.
+        
+        Args:
+            context: The browser context to set up
+            crawlerRunConfig: Configuration object containing all browser settings
+        """
+        # Set HTTP headers
+        if self.config.headers:
+            await context.set_extra_http_headers(self.config.headers)
+
+        # Add cookies
+        if self.config.cookies:
+            await context.add_cookies(self.config.cookies)
+
+        # Apply storage state if provided
+        if self.config.storage_state:
+            await context.storage_state(path=None)
+
+        # Configure downloads
+        if self.config.accept_downloads:
+            context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            if self.config.downloads_path:
+                context._impl_obj._options["accept_downloads"] = True
+                context._impl_obj._options["downloads_path"] = self.config.downloads_path
+
+        # Handle user agent and browser hints
+        if self.config.user_agent:
+            combined_headers = {
+                "User-Agent": self.config.user_agent,
+                "sec-ch-ua": self.config.browser_hint,
+            }
+            combined_headers.update(self.config.headers)
+            await context.set_extra_http_headers(combined_headers)
+
+        # Add default cookie
+        target_url = (crawlerRunConfig and crawlerRunConfig.url) or "https://crawl4ai.com/"
+        await context.add_cookies(
+            [
+                {
+                    "name": "cookiesEnabled",
+                    "value": "true",
+                    "url": target_url,
+                }
+            ]
+        )
+
+        # Handle navigator overrides
+        if crawlerRunConfig:
+            if (
+                crawlerRunConfig.override_navigator
+                or crawlerRunConfig.simulate_user
+                or crawlerRunConfig.magic
+            ):
+                await context.add_init_script(load_js_script("navigator_overrider"))
+                
+    async def kill_session(self, session_id: str):
+        """Kill a browser session and clean up resources.
+
+        Args:
+            session_id (str): The session ID to kill.
+        """
+        if session_id not in self.sessions:
+            return
+            
+        context, page, _ = self.sessions[session_id]
+        
+        # Close the page
+        try:
+            await page.close()
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error closing page for session {session_id}: {str(e)}", tag="BROWSER")
+        
+        # Remove session from tracking
+        del self.sessions[session_id]
+        
+        # Clean up any contexts that no longer have pages
+        await self._cleanup_unused_contexts()
+        
+        if self.logger:
+            self.logger.debug(f"Killed session: {session_id}", tag="BROWSER")
+
+    async def _cleanup_unused_contexts(self):
+        """Clean up contexts that no longer have any pages."""
+        async with self._contexts_lock:
+            # Get all contexts we're managing
+            contexts_to_check = list(self.contexts_by_config.values())
+            
+            for context in contexts_to_check:
+                # Check if the context has any pages left
+                if not context.pages:
+                    # No pages left, we can close this context
+                    config_signature = next((sig for sig, ctx in self.contexts_by_config.items() 
+                                           if ctx == context), None)
+                    if config_signature:
+                        try:
+                            await context.close()
+                            del self.contexts_by_config[config_signature]
+                            if self.logger:
+                                self.logger.debug(f"Closed unused context", tag="BROWSER")
+                        except Exception as e:
+                            if self.logger:
+                                self.logger.error(f"Error closing unused context: {str(e)}", tag="BROWSER")
+    
+    def _cleanup_expired_sessions(self):
+        """Clean up expired sessions based on TTL."""
+        current_time = time.time()
+        expired_sessions = [
+            sid
+            for sid, (_, _, last_used) in self.sessions.items()
+            if current_time - last_used > self.session_ttl
+        ]
+        
+        for sid in expired_sessions:
+            if self.logger:
+                self.logger.debug(f"Session expired: {sid}", tag="BROWSER")
+            asyncio.create_task(self.kill_session(sid))
+
+    async def close(self):
+        """Close the browser and clean up resources.
+        
+        This method handles common cleanup tasks like:
+        1. Persisting storage state if a user_data_dir is configured
+        2. Closing all sessions
+        3. Closing all browser contexts
+        4. Closing the browser
+        5. Stopping Playwright
+        
+        Child classes should override this method to add their specific cleanup logic,
+        but should call super().close() to ensure common cleanup tasks are performed.
+        """
+        # Set a flag to prevent race conditions during cleanup
+        self.shutting_down = True
+        
+        try:
+            # Add brief delay if configured
+            if self.config.sleep_on_close:
+                await asyncio.sleep(0.5)
+                
+            # Persist storage state if using a user data directory
+            if self.config.user_data_dir and self.browser:
+                for context in self.browser.contexts:
+                    try:
+                        # Ensure the directory exists
+                        storage_dir = os.path.join(self.config.user_data_dir, "Default")
+                        os.makedirs(storage_dir, exist_ok=True)
+                        
+                        # Save storage state
+                        storage_path = os.path.join(storage_dir, "storage_state.json")
+                        await context.storage_state(path=storage_path)
+                        
+                        if self.logger:
+                            self.logger.debug("Storage state persisted before closing browser", tag="BROWSER")
+                    except Exception as e:
+                        if self.logger:
+                            self.logger.warning(
+                                message="Failed to ensure storage persistence: {error}",
+                                tag="BROWSER", 
+                                params={"error": str(e)}
+                            )
+            
+            # Close all active sessions
+            session_ids = list(self.sessions.keys())
+            for session_id in session_ids:
+                await self.kill_session(session_id)
+                
+            # Close all cached contexts
+            for ctx in self.contexts_by_config.values():
+                try:
+                    await ctx.close()
+                except Exception as e:
+                    if self.logger:
+                        self.logger.error(
+                            message="Error closing context: {error}",
+                            tag="BROWSER",
+                            params={"error": str(e)}
+                        )
+            self.contexts_by_config.clear()
+            
+            # Close the browser if it exists
+            if self.browser:
+                await self.browser.close()
+                self.browser = None
+                
+            # Stop playwright
+            if self.playwright:
+                await self.playwright.stop()
+                self.playwright = None
+                
+        except Exception as e:
+            if self.logger:
+                self.logger.error(
+                    message="Error during browser cleanup: {error}",
+                    tag="BROWSER",
+                    params={"error": str(e)}
+                )
+        finally:
+            # Reset shutting down flag
+            self.shutting_down = False
+    
+    

crawl4ai/browser/strategies/builtin.py

@@ -0,0 +1,468 @@
+import asyncio
+import os
+import time
+import json
+import subprocess
+import shutil
+import signal
+from typing import Optional, Dict, Any, Tuple
+
+
+from ...async_logger import AsyncLogger
+from ...async_configs import CrawlerRunConfig
+from playwright.async_api import Page, BrowserContext
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig
+from ...utils import get_home_folder
+from ..utils import get_browser_executable, is_windows, is_browser_running, find_process_by_port, terminate_process
+
+
+from .cdp import CDPBrowserStrategy
+from .base import BaseBrowserStrategy
+
+class BuiltinBrowserStrategy(CDPBrowserStrategy):
+    """Built-in browser strategy.
+    
+    This strategy extends the CDP strategy to use the built-in browser.
+    """
+    
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the built-in browser strategy.
+        
+        Args:
+            config: Browser configuration
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+        self.builtin_browser_dir = os.path.join(get_home_folder(), "builtin-browser") if not self.config.user_data_dir else self.config.user_data_dir
+        self.builtin_config_file = os.path.join(self.builtin_browser_dir, "browser_config.json")
+
+        # Raise error if user data dir is already engaged
+        if self._check_user_dir_is_engaged(self.builtin_browser_dir):
+            raise Exception(f"User data directory {self.builtin_browser_dir} is already engaged by another browser instance.")
+
+        os.makedirs(self.builtin_browser_dir, exist_ok=True)
+    
+    def _check_user_dir_is_engaged(self, user_data_dir: str) -> bool:
+        """Check if the user data directory is already in use.
+        
+        Returns:
+            bool: True if the directory is engaged, False otherwise
+        """
+        # Load browser config file, then iterate in port_map values, check "user_data_dir" key if it matches
+        # the current user data directory
+        if os.path.exists(self.builtin_config_file):
+            try:
+                with open(self.builtin_config_file, 'r') as f:
+                    browser_info_dict = json.load(f)
+                
+                # Check if user data dir is already engaged
+                for port_str, browser_info in browser_info_dict.get("port_map", {}).items():
+                    if browser_info.get("user_data_dir") == user_data_dir:
+                        return True
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(f"Error reading built-in browser config: {str(e)}", tag="BUILTIN")
+        return False
+
+    async def start(self):
+        """Start or connect to the built-in browser.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Initialize Playwright instance via base class method
+        await BaseBrowserStrategy.start(self)
+        
+        try:
+            # Check for existing built-in browser (get_browser_info already checks if running)
+            browser_info = self.get_browser_info()
+            if browser_info:
+                if self.logger:
+                    self.logger.info(f"Using existing built-in browser at {browser_info.get('cdp_url')}", tag="BROWSER")
+                self.config.cdp_url = browser_info.get('cdp_url')
+            else:
+                if self.logger:
+                    self.logger.info("Built-in browser not found, launching new instance...", tag="BROWSER")
+                cdp_url = await self.launch_builtin_browser(
+                    browser_type=self.config.browser_type,
+                    debugging_port=self.config.debugging_port,
+                    headless=self.config.headless,
+                )
+                if not cdp_url:
+                    if self.logger:
+                        self.logger.warning("Failed to launch built-in browser, falling back to regular CDP strategy", tag="BROWSER")
+                    # Call CDP's start but skip BaseBrowserStrategy.start() since we already called it
+                    return await CDPBrowserStrategy.start(self)
+                self.config.cdp_url = cdp_url
+            
+            # Connect to the browser using CDP protocol
+            self.browser = await self.playwright.chromium.connect_over_cdp(self.config.cdp_url)
+            
+            # Get or create default context
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+            else:
+                self.default_context = await self.create_browser_context()
+            
+            await self.setup_context(self.default_context)
+            
+            if self.logger:
+                self.logger.debug(f"Connected to built-in browser at {self.config.cdp_url}", tag="BUILTIN")
+                
+            return self
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Failed to start built-in browser: {str(e)}", tag="BUILTIN")
+            
+            # There is a possibility that at this point I need to clean up some resourece
+            raise
+
+    def _get_builtin_browser_info(cls, debugging_port: int, config_file: str, logger: Optional[AsyncLogger] = None) -> Optional[Dict[str, Any]]:
+        """Get information about the built-in browser for a specific debugging port.
+        
+        Args:
+            debugging_port: The debugging port to look for
+            config_file: Path to the config file
+            logger: Optional logger for recording events
+            
+        Returns:
+            dict: Browser information or None if no running browser is configured for this port
+        """
+        if not os.path.exists(config_file):
+            return None
+            
+        try:
+            with open(config_file, 'r') as f:
+                browser_info_dict = json.load(f)
+            
+            # Get browser info from port map
+            if isinstance(browser_info_dict, dict) and "port_map" in browser_info_dict:
+                port_str = str(debugging_port)
+                if port_str in browser_info_dict["port_map"]:
+                    browser_info = browser_info_dict["port_map"][port_str]
+                    
+                    # Check if the browser is still running
+                    pids = browser_info.get('pid', '')
+                    if isinstance(pids, str):
+                        pids = [int(pid) for pid in pids.split() if pid.isdigit()]
+                    elif isinstance(pids, int):
+                        pids = [pids]
+                    else:
+                        pids = []
+
+                    # Check if any of the PIDs are running
+                    if not pids:
+                        if logger:
+                            logger.warning(f"Built-in browser on port {debugging_port} has no valid PID", tag="BUILTIN")
+                        # Remove this port from the dictionary
+                        del browser_info_dict["port_map"][port_str]
+                        with open(config_file, 'w') as f:
+                            json.dump(browser_info_dict, f, indent=2)
+                        return None
+                    # Check if any of the PIDs are running
+                    for pid in pids:
+                        if is_browser_running(pid):
+                            browser_info['pid'] = pid
+                            break
+                    else:
+                        # If none of the PIDs are running, remove this port from the dictionary
+                        if logger:
+                            logger.warning(f"Built-in browser on port {debugging_port} is not running", tag="BUILTIN")
+                        # Remove this port from the dictionary
+                        del browser_info_dict["port_map"][port_str]
+                        with open(config_file, 'w') as f:
+                            json.dump(browser_info_dict, f, indent=2)
+                        return None
+                    
+                    return browser_info
+            
+            return None
+                
+        except Exception as e:
+            if logger:
+                logger.error(f"Error reading built-in browser config: {str(e)}", tag="BUILTIN")
+            return None
+            
+    def get_browser_info(self) -> Optional[Dict[str, Any]]:
+        """Get information about the current built-in browser instance.
+        
+        Returns:
+            dict: Browser information or None if no running browser is configured
+        """
+        return self._get_builtin_browser_info(
+            debugging_port=self.config.debugging_port,
+            config_file=self.builtin_config_file,
+            logger=self.logger
+        )
+    
+    async def launch_builtin_browser(self, 
+                               browser_type: str = "chromium",
+                               debugging_port: int = 9222,
+                               headless: bool = True) -> Optional[str]:
+        """Launch a browser in the background for use as the built-in browser.
+        
+        Args:
+            browser_type: Type of browser to launch ('chromium' or 'firefox')
+            debugging_port: Port to use for CDP debugging
+            headless: Whether to run in headless mode
+            
+        Returns:
+            str: CDP URL for the browser, or None if launch failed
+        """
+        # Check if there's an existing browser still running
+        browser_info = self._get_builtin_browser_info(
+            debugging_port=debugging_port,
+            config_file=self.builtin_config_file,
+            logger=self.logger
+        )
+        if browser_info:
+            if self.logger:
+                self.logger.info(f"Built-in browser is already running on port {debugging_port}", tag="BUILTIN")
+            return browser_info.get('cdp_url')
+        
+        # Create a user data directory for the built-in browser
+        user_data_dir = os.path.join(self.builtin_browser_dir, "user_data")
+     
+        # Raise error if user data dir is already engaged
+        if self._check_user_dir_is_engaged(user_data_dir):
+            raise Exception(f"User data directory {user_data_dir} is already engaged by another browser instance.")
+            
+        # Create the user data directory if it doesn't exist
+        os.makedirs(user_data_dir, exist_ok=True)
+        
+        # Prepare browser launch arguments
+        browser_args = super()._build_browser_args()
+        browser_path = await get_browser_executable(browser_type)
+        base_args = [browser_path]
+
+        if browser_type == "chromium":
+            args = [
+                browser_path,
+                f"--remote-debugging-port={debugging_port}",
+                f"--user-data-dir={user_data_dir}",
+            ]
+            # if headless:
+            #     args.append("--headless=new")
+
+        elif browser_type == "firefox":
+            args = [
+                browser_path,
+                "--remote-debugging-port",
+                str(debugging_port),
+                "--profile",
+                user_data_dir,
+            ]
+            if headless:
+                args.append("--headless")
+        else:
+            if self.logger:
+                self.logger.error(f"Browser type {browser_type} not supported for built-in browser", tag="BUILTIN")
+            return None
+        
+        args = base_args + browser_args + args
+        
+        try:
+
+            # Check if the port is already in use
+            PID = ""
+            cdp_url = f"http://localhost:{debugging_port}"
+            config_json = await self._check_port_in_use(cdp_url)
+            if config_json:
+                if self.logger:
+                    self.logger.info(f"Port {debugging_port} is already in use.", tag="BUILTIN")
+                PID = find_process_by_port(debugging_port)
+            else:
+                # Start the browser process detached
+                process = None
+                if is_windows():
+                    process = subprocess.Popen(
+                        args, 
+                        stdout=subprocess.PIPE, 
+                        stderr=subprocess.PIPE,
+                        creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
+                    )
+                else:
+                    process = subprocess.Popen(
+                        args, 
+                        stdout=subprocess.PIPE, 
+                        stderr=subprocess.PIPE,
+                        preexec_fn=os.setpgrp  # Start in a new process group
+                    )
+                
+                # Wait briefly to ensure the process starts successfully
+                await asyncio.sleep(2.0)
+                
+                # Check if the process is still running
+                if process and process.poll() is not None:
+                    if self.logger:
+                        self.logger.error(f"Browser process exited immediately with code {process.returncode}", tag="BUILTIN")
+                    return None
+            
+                PID = process.pid
+                # Construct CDP URL
+                config_json = await self._check_port_in_use(cdp_url)
+
+            
+            # Create browser info
+            browser_info = {
+                'pid': PID,
+                'cdp_url': cdp_url,
+                'user_data_dir': user_data_dir,
+                'browser_type': browser_type,
+                'debugging_port': debugging_port,
+                'start_time': time.time(),
+                'config': config_json
+            }
+            
+            # Read existing config file if it exists
+            port_map = {}
+            if os.path.exists(self.builtin_config_file):
+                try:
+                    with open(self.builtin_config_file, 'r') as f:
+                        existing_data = json.load(f)
+                    
+                    # Check if it already uses port mapping
+                    if isinstance(existing_data, dict) and "port_map" in existing_data:
+                        port_map = existing_data["port_map"]
+
+                    # # Convert legacy format to port mapping
+                    # elif isinstance(existing_data, dict) and "debugging_port" in existing_data:
+                    #     old_port = str(existing_data.get("debugging_port"))
+                    #     if self._is_browser_running(existing_data.get("pid")):
+                    #         port_map[old_port] = existing_data
+                except Exception as e:
+                    if self.logger:
+                        self.logger.warning(f"Could not read existing config: {str(e)}", tag="BUILTIN")
+            
+            # Add/update this browser in the port map
+            port_map[str(debugging_port)] = browser_info
+            
+            # Write updated config
+            with open(self.builtin_config_file, 'w') as f:
+                json.dump({"port_map": port_map}, f, indent=2)
+                
+            # Detach from the browser process - don't keep any references
+            # This is important to allow the Python script to exit while the browser continues running
+            process = None
+                
+            if self.logger:
+                self.logger.success(f"Built-in browser launched at CDP URL: {cdp_url}", tag="BUILTIN")
+            return cdp_url
+            
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Error launching built-in browser: {str(e)}", tag="BUILTIN")
+            return None
+
+    async def _check_port_in_use(self, cdp_url: str) -> dict:
+        """Check if a port is already in use by a Chrome DevTools instance.
+        
+        Args:
+            cdp_url: The CDP URL to check
+            
+        Returns:
+            dict: Chrome DevTools protocol version information or None if not found
+        """
+        import aiohttp
+        json_url = f"{cdp_url}/json/version"
+        json_config = None
+        
+        try:
+            async with aiohttp.ClientSession() as session:
+                try:
+                    async with session.get(json_url, timeout=2.0) as response:
+                        if response.status == 200:
+                            json_config = await response.json()
+                            if self.logger:
+                                self.logger.debug(f"Found CDP server running at {cdp_url}", tag="BUILTIN")
+                            return json_config
+                except (aiohttp.ClientError, asyncio.TimeoutError):
+                    pass
+            return None
+        except Exception as e:
+            if self.logger:
+                self.logger.debug(f"Error checking CDP port: {str(e)}", tag="BUILTIN")
+            return None
+
+    async def kill_builtin_browser(self) -> bool:
+        """Kill the built-in browser if it's running.
+        
+        Returns:
+            bool: True if the browser was killed, False otherwise
+        """
+        browser_info = self.get_browser_info()
+        if not browser_info:
+            if self.logger:
+                self.logger.warning(f"No built-in browser found on port {self.config.debugging_port}", tag="BUILTIN")
+            return False
+            
+        pid = browser_info.get('pid')
+        if not pid:
+            return False
+            
+        success, error_msg = terminate_process(pid, logger=self.logger)
+        if success:
+            # Update config file to remove this browser
+            with open(self.builtin_config_file, 'r') as f:
+                browser_info_dict = json.load(f)
+
+            # Remove this port from the dictionary
+            port_str = str(self.config.debugging_port)
+            if port_str in browser_info_dict.get("port_map", {}):
+                del browser_info_dict["port_map"][port_str]
+
+            with open(self.builtin_config_file, 'w') as f:
+                json.dump(browser_info_dict, f, indent=2)
+
+            # Remove user data directory if it exists
+            if os.path.exists(self.builtin_browser_dir):
+                shutil.rmtree(self.builtin_browser_dir)
+
+            # Clear the browser info cache
+            self.browser = None
+            self.temp_dir = None
+            self.shutting_down = True
+                
+            if self.logger:
+                self.logger.success("Built-in browser terminated", tag="BUILTIN")
+            return True
+        else:
+            if self.logger:
+                self.logger.error(f"Error killing built-in browser: {error_msg}", tag="BUILTIN")
+            return False
+    
+    async def get_builtin_browser_status(self) -> Dict[str, Any]:
+        """Get status information about the built-in browser.
+        
+        Returns:
+            dict: Status information with running, cdp_url, and info fields
+        """
+        browser_info = self.get_browser_info()
+        
+        if not browser_info:
+            return {
+                'running': False,
+                'cdp_url': None,
+                'info': None,
+                'port': self.config.debugging_port
+            }
+            
+        return {
+            'running': True,
+            'cdp_url': browser_info.get('cdp_url'),
+            'info': browser_info,
+            'port': self.config.debugging_port
+        }
+
+    async def close(self):
+        """Close the built-in browser and clean up resources."""
+        # Call parent class close method
+        await super().close()
+        
+        # Clean up built-in browser if we created it and were in shutdown mode
+        if self.shutting_down:
+            await self.kill_builtin_browser()
+            if self.logger:
+                self.logger.debug("Killed built-in browser during shutdown", tag="BUILTIN")

crawl4ai/browser/strategies/cdp.py

@@ -0,0 +1,281 @@
+"""Browser strategies module for Crawl4AI.
+
+This module implements the browser strategy pattern for different
+browser implementations, including Playwright, CDP, and builtin browsers.
+"""
+
+import asyncio
+import os
+import time
+import json
+import subprocess
+import shutil
+from typing import Optional, Tuple, List
+
+from playwright.async_api import BrowserContext, Page
+
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig, CrawlerRunConfig
+from ..utils import get_playwright, get_browser_executable, create_temp_directory, is_windows, check_process_is_running, terminate_process
+
+from .base import BaseBrowserStrategy
+
+class CDPBrowserStrategy(BaseBrowserStrategy):
+    """CDP-based browser strategy.
+    
+    This strategy connects to an existing browser using CDP protocol or
+    launches and connects to a browser using CDP.
+    """
+    
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the CDP browser strategy.
+        
+        Args:
+            config: Browser configuration
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+        self.browser_process = None
+        self.temp_dir = None
+        self.shutting_down = False
+        
+    async def start(self):
+        """Start or connect to the browser using CDP.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Call the base class start to initialize Playwright
+        await super().start()
+        
+        try:
+            # Get or create CDP URL
+            cdp_url = await self._get_or_create_cdp_url()
+            
+            # Connect to the browser using CDP
+            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
+            
+            # Get or create default context
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+            else:
+                self.default_context = await self.create_browser_context()
+            
+            await self.setup_context(self.default_context)
+            
+            if self.logger:
+                self.logger.debug(f"Connected to CDP browser at {cdp_url}", tag="CDP")
+
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Failed to connect to CDP browser: {str(e)}", tag="CDP")
+
+            # Clean up any resources before re-raising
+            await self._cleanup_process()
+            raise
+            
+        return self
+    
+    async def _get_or_create_cdp_url(self) -> str:
+        """Get existing CDP URL or launch a browser and return its CDP URL.
+        
+        Returns:
+            str: CDP URL for connecting to the browser
+        """
+        # If CDP URL is provided, just return it
+        if self.config.cdp_url:
+            return self.config.cdp_url
+
+        # Create temp dir if needed
+        if not self.config.user_data_dir:
+            self.temp_dir = create_temp_directory()
+            user_data_dir = self.temp_dir
+        else:
+            user_data_dir = self.config.user_data_dir
+
+        # Get browser args based on OS and browser type
+        # args = await self._get_browser_args(user_data_dir)
+        browser_args = super()._build_browser_args()
+        browser_path = await get_browser_executable(self.config.browser_type)
+        base_args = [browser_path]
+
+        if self.config.browser_type == "chromium":
+            args = [
+                f"--remote-debugging-port={self.config.debugging_port}",
+                f"--user-data-dir={user_data_dir}",
+            ]
+            # if self.config.headless:
+            #     args.append("--headless=new")
+
+        elif self.config.browser_type == "firefox":
+            args = [
+                "--remote-debugging-port",
+                str(self.config.debugging_port),
+                "--profile",
+                user_data_dir,
+            ]
+            if self.config.headless:
+                args.append("--headless")
+        else:
+            raise NotImplementedError(f"Browser type {self.config.browser_type} not supported")
+
+        args = base_args + browser_args['args'] + args         
+
+        # Start browser process
+        try:
+            # Use DETACHED_PROCESS flag on Windows to fully detach the process
+            # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group
+            if is_windows():
+                self.browser_process = subprocess.Popen(
+                    args, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
+                )
+            else:
+                self.browser_process = subprocess.Popen(
+                    args, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    preexec_fn=os.setpgrp  # Start in a new process group
+                )
+                
+            # Monitor for a short time to make sure it starts properly
+            is_running, return_code, stdout, stderr = await check_process_is_running(self.browser_process, delay=2)
+            if not is_running:
+                if self.logger:
+                    self.logger.error(
+                        message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
+                        tag="ERROR",
+                        params={
+                            "code": return_code,
+                            "stdout": stdout.decode() if stdout else "",
+                            "stderr": stderr.decode() if stderr else "",
+                        },
+                    )
+                await self._cleanup_process()
+                raise Exception("Browser process terminated unexpectedly")
+
+            return f"http://localhost:{self.config.debugging_port}"
+        except Exception as e:
+            await self._cleanup_process()
+            raise Exception(f"Failed to start browser: {e}")    
+
+    async def _cleanup_process(self):
+        """Cleanup browser process and temporary directory."""
+        # Set shutting_down flag BEFORE any termination actions
+        self.shutting_down = True
+
+        if self.browser_process:
+            try:
+                # Only attempt termination if the process is still running
+                if self.browser_process.poll() is None:
+                    # Use our robust cross-platform termination utility
+                    success = terminate_process(
+                        pid=self.browser_process.pid,
+                        timeout=1.0,  # Equivalent to the previous 10*0.1s wait
+                        logger=self.logger
+                    )
+                    
+                    if not success and self.logger:
+                        self.logger.warning(
+                            message="Failed to terminate browser process cleanly",
+                            tag="PROCESS"
+                        )
+                        
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(
+                        message="Error during browser process cleanup: {error}",
+                        tag="ERROR",
+                        params={"error": str(e)},
+                    )
+
+        if self.temp_dir and os.path.exists(self.temp_dir):
+            try:
+                shutil.rmtree(self.temp_dir)
+                self.temp_dir = None
+                if self.logger:
+                    self.logger.debug("Removed temporary directory", tag="CDP")
+            except Exception as e:
+                if self.logger:
+                    self.logger.error(
+                        message="Error removing temporary directory: {error}",
+                        tag="CDP",
+                        params={"error": str(e)}
+                    )
+
+        self.browser_process = None
+    
+    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        # For CDP, we typically use the shared default_context
+        context = self.default_context
+        pages = context.pages
+        
+        # Otherwise, check if we have an existing context for this config
+        config_signature = self._make_config_signature(crawlerRunConfig)
+        self.contexts_by_config[config_signature] = context
+
+        await self.setup_context(context, crawlerRunConfig)
+
+        # Check if there's already a page with the target URL
+        page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+        
+        # If not found, create a new page
+        if not page:
+            page = await context.new_page()
+        
+        return page, context
+
+    async def _get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page for the given configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Call parent method to ensure browser is started
+        await super().get_page(crawlerRunConfig)
+        
+        # For CDP, we typically use the shared default_context
+        context = self.default_context
+        pages = context.pages
+        
+        # Otherwise, check if we have an existing context for this config
+        config_signature = self._make_config_signature(crawlerRunConfig)
+        self.contexts_by_config[config_signature] = context
+
+        await self.setup_context(context, crawlerRunConfig)
+
+        # Check if there's already a page with the target URL
+        page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+        
+        # If not found, create a new page
+        if not page:
+            page = await context.new_page()
+        
+        # If a session_id is specified, store this session for reuse
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+        
+        return page, context
+
+    async def close(self):
+        """Close the CDP browser and clean up resources."""
+        # Skip cleanup if using external CDP URL and not launched by us
+        if self.config.cdp_url and not self.browser_process:
+            if self.logger:
+                self.logger.debug("Skipping cleanup for external CDP browser", tag="CDP")
+            return
+        
+        # Call parent implementation for common cleanup
+        await super().close()
+        
+        # Additional CDP-specific cleanup
+        await asyncio.sleep(0.5)
+        await self._cleanup_process()

crawl4ai/browser/strategies/docker_strategy.py

@@ -0,0 +1,430 @@
+"""Docker browser strategy module for Crawl4AI.
+
+This module provides browser strategies for running browsers in Docker containers,
+which offers better isolation, consistency across platforms, and easy scaling.
+"""
+
+import os
+import uuid
+from typing import List, Optional
+
+
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig
+from ..models import DockerConfig
+from ..docker_registry import DockerRegistry
+from ..docker_utils import DockerUtils
+from .builtin import CDPBrowserStrategy
+from .base import BaseBrowserStrategy
+
+class DockerBrowserStrategy(CDPBrowserStrategy):
+    """Docker-based browser strategy.
+
+    Extends the CDPBrowserStrategy to run browsers in Docker containers.
+    Supports two modes:
+    1. "connect" - Uses a Docker image with Chrome already running
+    2. "launch" - Starts Chrome within the container with custom settings
+
+    Attributes:
+        docker_config: Docker-specific configuration options
+        container_id: ID of current Docker container
+        container_name: Name assigned to the container
+        registry: Registry for tracking and reusing containers
+        docker_utils: Utilities for Docker operations
+        chrome_process_id: Process ID of Chrome within container
+        socat_process_id: Process ID of socat within container
+        internal_cdp_port: Chrome's internal CDP port
+        internal_mapped_port: Port that socat maps to internally
+    """
+
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the Docker browser strategy.
+
+        Args:
+            config: Browser configuration including Docker-specific settings
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+
+        # Initialize Docker-specific attributes
+        self.docker_config = self.config.docker_config or DockerConfig()
+        self.container_id = None
+        self.container_name = f"crawl4ai-browser-{uuid.uuid4().hex[:8]}"
+
+        # Use the shared registry file path for consistency with BuiltinBrowserStrategy
+        registry_file = self.docker_config.registry_file
+        if registry_file is None and self.config.user_data_dir:
+            # Use the same registry file as BuiltinBrowserStrategy if possible
+            registry_file = os.path.join(
+                os.path.dirname(self.config.user_data_dir), "browser_config.json"
+            )
+
+        self.registry = DockerRegistry(self.docker_config.registry_file)
+        self.docker_utils = DockerUtils(logger)
+        self.chrome_process_id = None
+        self.socat_process_id = None
+        self.internal_cdp_port = 9222  # Chrome's internal CDP port
+        self.internal_mapped_port = 9223  # Port that socat maps to internally
+        self.shutting_down = False
+
+    async def start(self):
+        """Start or connect to a browser running in a Docker container.
+
+        This method initializes Playwright and establishes a connection to
+        a browser running in a Docker container. Depending on the configured mode:
+        - "connect": Connects to a container with Chrome already running
+        - "launch": Creates a container and launches Chrome within it
+
+        Returns:
+            self: For method chaining
+        """
+        # Initialize Playwright
+        await BaseBrowserStrategy.start(self)
+
+        if self.logger:
+            self.logger.info(
+                f"Starting Docker browser strategy in {self.docker_config.mode} mode",
+                tag="DOCKER",
+            )
+
+        try:
+            # Get CDP URL by creating or reusing a Docker container
+            # This handles the container management and browser startup
+            cdp_url = await self._get_or_create_cdp_url()
+
+            if not cdp_url:
+                raise Exception(
+                    "Failed to establish CDP connection to Docker container"
+                )
+
+            if self.logger:
+                self.logger.info(
+                    f"Connecting to browser in Docker via CDP: {cdp_url}", tag="DOCKER"
+                )
+
+            # Connect to the browser using CDP
+            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
+
+            # Get existing context or create default context
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+                if self.logger:
+                    self.logger.debug("Using existing browser context", tag="DOCKER")
+            else:
+                if self.logger:
+                    self.logger.debug("Creating new browser context", tag="DOCKER")
+                self.default_context = await self.create_browser_context()
+                await self.setup_context(self.default_context)
+
+            return self
+
+        except Exception as e:
+            # Clean up resources if startup fails
+            if self.container_id and not self.docker_config.persistent:
+                if self.logger:
+                    self.logger.warning(
+                        f"Cleaning up container after failed start: {self.container_id[:12]}",
+                        tag="DOCKER",
+                    )
+                await self.docker_utils.remove_container(self.container_id)
+                self.registry.unregister_container(self.container_id)
+                self.container_id = None
+
+            if self.playwright:
+                await self.playwright.stop()
+                self.playwright = None
+
+            # Re-raise the exception
+            if self.logger:
+                self.logger.error(
+                    f"Failed to start Docker browser: {str(e)}", tag="DOCKER"
+                )
+            raise
+
+    async def _generate_config_hash(self) -> str:
+        """Generate a hash of the configuration for container matching.
+
+        Returns:
+            Hash string uniquely identifying this configuration
+        """
+        # Create a dict with the relevant parts of the config
+        config_dict = {
+            "image": self.docker_config.image,
+            "mode": self.docker_config.mode,
+            "browser_type": self.config.browser_type,
+            "headless": self.config.headless,
+        }
+
+        # Add browser-specific config if in launch mode
+        if self.docker_config.mode == "launch":
+            config_dict.update(
+                {
+                    "text_mode": self.config.text_mode,
+                    "light_mode": self.config.light_mode,
+                    "viewport_width": self.config.viewport_width,
+                    "viewport_height": self.config.viewport_height,
+                }
+            )
+
+        # Use the utility method to generate the hash
+        return self.docker_utils.generate_config_hash(config_dict)
+
+    async def _get_or_create_cdp_url(self) -> str:
+        """Get CDP URL by either creating a new container or using an existing one.
+
+        Returns:
+            CDP URL for connecting to the browser
+
+        Raises:
+            Exception: If container creation or browser launch fails
+        """
+        # If CDP URL is explicitly provided, use it
+        if self.config.cdp_url:
+            return self.config.cdp_url
+
+        # Ensure Docker image exists (will build if needed)
+        image_name = await self.docker_utils.ensure_docker_image_exists(
+            self.docker_config.image, self.docker_config.mode
+        )
+
+        # Generate config hash for container matching
+        config_hash = await self._generate_config_hash()
+
+        # Look for existing container with matching config
+        container_id = await self.registry.find_container_by_config(
+            config_hash, self.docker_utils
+        )
+
+        if container_id:
+            # Use existing container
+            self.container_id = container_id
+            host_port = self.registry.get_container_host_port(container_id)
+            if self.logger:
+                self.logger.info(
+                    f"Using existing Docker container: {container_id[:12]}",
+                    tag="DOCKER",
+                )
+        else:
+            # Get a port for the new container
+            host_port = (
+                self.docker_config.host_port
+                or self.registry.get_next_available_port(self.docker_utils)
+            )
+
+            # Prepare volumes list
+            volumes = list(self.docker_config.volumes)
+
+            # Add user data directory if specified
+            if self.docker_config.user_data_dir:
+                # Ensure user data directory exists
+                os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
+                volumes.append(
+                    f"{self.docker_config.user_data_dir}:{self.docker_config.container_user_data_dir}"
+                )
+
+                # # Update config user_data_dir to point to container path
+                # self.config.user_data_dir = self.docker_config.container_user_data_dir
+
+            # Create a new container
+            container_id = await self.docker_utils.create_container(
+                image_name=image_name,
+                host_port=host_port,
+                container_name=self.container_name,
+                volumes=volumes,
+                network=self.docker_config.network,
+                env_vars=self.docker_config.env_vars,
+                cpu_limit=self.docker_config.cpu_limit,
+                memory_limit=self.docker_config.memory_limit,
+                extra_args=self.docker_config.extra_args,
+            )
+
+            if not container_id:
+                raise Exception("Failed to create Docker container")
+
+            self.container_id = container_id
+
+            # Wait for container to be ready
+            await self.docker_utils.wait_for_container_ready(container_id)
+
+            # Handle specific setup based on mode
+            if self.docker_config.mode == "launch":
+                # In launch mode, we need to start socat and Chrome
+                await self.docker_utils.start_socat_in_container(container_id)
+
+                # Build browser arguments
+                browser_args = self._build_browser_args()
+
+                # Launch Chrome
+                await self.docker_utils.launch_chrome_in_container(
+                    container_id, browser_args
+                )
+
+                # Get PIDs for later cleanup
+                self.chrome_process_id = (
+                    await self.docker_utils.get_process_id_in_container(
+                        container_id, "chromium"
+                    )
+                )
+                self.socat_process_id = (
+                    await self.docker_utils.get_process_id_in_container(
+                        container_id, "socat"
+                    )
+                )
+
+            # Wait for CDP to be ready
+            cdp_json_config = await self.docker_utils.wait_for_cdp_ready(host_port)
+
+            if cdp_json_config:
+                # Register the container in the shared registry
+                self.registry.register_container(
+                    container_id, host_port, config_hash, cdp_json_config
+                )
+            else:
+                raise Exception("Failed to get CDP JSON config from Docker container")
+
+            if self.logger:
+                self.logger.success(
+                    f"Docker container ready: {container_id[:12]} on port {host_port}",
+                    tag="DOCKER",
+                )
+
+        # Return CDP URL
+        return f"http://localhost:{host_port}"
+
+    def _build_browser_args(self) -> List[str]:
+        """Build Chrome command line arguments based on BrowserConfig.
+
+        Returns:
+            List of command line arguments for Chrome
+        """
+        # Call parent method to get common arguments
+        browser_args = super()._build_browser_args()
+        return browser_args["args"] + [
+            f"--remote-debugging-port={self.internal_cdp_port}",
+            "--remote-debugging-address=0.0.0.0",  # Allow external connections
+            "--disable-dev-shm-usage",
+            "--headless=new",
+        ]
+
+        # args = [
+        #     "--no-sandbox",
+        #     "--disable-gpu",
+        #     f"--remote-debugging-port={self.internal_cdp_port}",
+        #     "--remote-debugging-address=0.0.0.0",  # Allow external connections
+        #     "--disable-dev-shm-usage",
+        # ]
+
+        # if self.config.headless:
+        #     args.append("--headless=new")
+
+        # if self.config.viewport_width and self.config.viewport_height:
+        #     args.append(f"--window-size={self.config.viewport_width},{self.config.viewport_height}")
+
+        # if self.config.user_agent:
+        #     args.append(f"--user-agent={self.config.user_agent}")
+
+        # if self.config.text_mode:
+        #     args.extend([
+        #         "--blink-settings=imagesEnabled=false",
+        #         "--disable-remote-fonts",
+        #         "--disable-images",
+        #         "--disable-javascript",
+        #     ])
+
+        # if self.config.light_mode:
+        #     # Import here to avoid circular import
+        #     from ..utils import get_browser_disable_options
+        #     args.extend(get_browser_disable_options())
+
+        # if self.config.user_data_dir:
+        #     args.append(f"--user-data-dir={self.config.user_data_dir}")
+
+        # if self.config.extra_args:
+        #     args.extend(self.config.extra_args)
+
+        # return args
+
+    async def close(self):
+        """Close the browser and clean up Docker container if needed."""
+        # Set flag to track if we were the ones initiating shutdown
+        initiated_shutdown = not self.shutting_down
+        # Storage persistence for Docker needs special handling
+        # We need to store state before calling super().close() which will close the browser
+        if (
+            self.browser
+            and self.docker_config.user_data_dir
+            and self.docker_config.persistent
+        ):
+            for context in self.browser.contexts:
+                try:
+                    # Ensure directory exists
+                    os.makedirs(self.docker_config.user_data_dir, exist_ok=True)
+
+                    # Save storage state to user data directory
+                    storage_path = os.path.join(
+                        self.docker_config.user_data_dir, "storage_state.json"
+                    )
+                    await context.storage_state(path=storage_path)
+                    if self.logger:
+                        self.logger.debug(
+                            "Persisted Docker-specific storage state", tag="DOCKER"
+                        )
+                except Exception as e:
+                    if self.logger:
+                        self.logger.warning(
+                            message="Failed to persist Docker storage state: {error}",
+                            tag="DOCKER",
+                            params={"error": str(e)},
+                        )
+
+        # Call parent method to handle common cleanup
+        await super().close()
+
+        # Only perform container cleanup if we initiated shutdown
+        # and we need to handle Docker-specific resources
+        if initiated_shutdown:
+            # Only clean up container if not persistent
+            if self.container_id and not self.docker_config.persistent:
+                # Stop Chrome process in "launch" mode
+                if self.docker_config.mode == "launch" and self.chrome_process_id:
+                    await self.docker_utils.stop_process_in_container(
+                        self.container_id, self.chrome_process_id
+                    )
+                    if self.logger:
+                        self.logger.debug(
+                            f"Stopped Chrome process {self.chrome_process_id} in container",
+                            tag="DOCKER",
+                        )
+
+                # Stop socat process in "launch" mode
+                if self.docker_config.mode == "launch" and self.socat_process_id:
+                    await self.docker_utils.stop_process_in_container(
+                        self.container_id, self.socat_process_id
+                    )
+                    if self.logger:
+                        self.logger.debug(
+                            f"Stopped socat process {self.socat_process_id} in container",
+                            tag="DOCKER",
+                        )
+
+                # Remove or stop container based on configuration
+                if self.docker_config.remove_on_exit:
+                    await self.docker_utils.remove_container(self.container_id)
+                    # Unregister from registry
+                    if hasattr(self, "registry") and self.registry:
+                        self.registry.unregister_container(self.container_id)
+                    if self.logger:
+                        self.logger.debug(
+                            f"Removed Docker container {self.container_id}",
+                            tag="DOCKER",
+                        )
+                else:
+                    await self.docker_utils.stop_container(self.container_id)
+                    if self.logger:
+                        self.logger.debug(
+                            f"Stopped Docker container {self.container_id}",
+                            tag="DOCKER",
+                        )
+
+                self.container_id = None

crawl4ai/browser/strategies/playwright.py

@@ -0,0 +1,134 @@
+"""Browser strategies module for Crawl4AI.
+
+This module implements the browser strategy pattern for different
+browser implementations, including Playwright, CDP, and builtin browsers.
+"""
+
+import time
+from typing import Optional, Tuple
+
+from playwright.async_api import BrowserContext, Page
+
+from ...async_logger import AsyncLogger
+from ...async_configs import BrowserConfig, CrawlerRunConfig
+
+from playwright_stealth import StealthConfig
+
+from .base import BaseBrowserStrategy
+
+stealth_config = StealthConfig(
+    webdriver=True,
+    chrome_app=True,
+    chrome_csi=True,
+    chrome_load_times=True,
+    chrome_runtime=True,
+    navigator_languages=True,
+    navigator_plugins=True,
+    navigator_permissions=True,
+    webgl_vendor=True,
+    outerdimensions=True,
+    navigator_hardware_concurrency=True,
+    media_codecs=True,
+)
+
+class PlaywrightBrowserStrategy(BaseBrowserStrategy):
+    """Standard Playwright browser strategy.
+    
+    This strategy launches a new browser instance using Playwright
+    and manages browser contexts.
+    """
+    
+    def __init__(self, config: BrowserConfig, logger: Optional[AsyncLogger] = None):
+        """Initialize the Playwright browser strategy.
+        
+        Args:
+            config: Browser configuration
+            logger: Logger for recording events and errors
+        """
+        super().__init__(config, logger)
+        # No need to re-initialize sessions and session_ttl as they're now in the base class
+    
+    async def start(self):
+        """Start the browser instance.
+        
+        Returns:
+            self: For method chaining
+        """
+        # Call the base class start to initialize Playwright
+        await super().start()
+        
+        # Build browser arguments using the base class method
+        browser_args = self._build_browser_args()
+        
+        try:
+            # Launch appropriate browser type
+            if self.config.browser_type == "firefox":
+                self.browser = await self.playwright.firefox.launch(**browser_args)
+            elif self.config.browser_type == "webkit":
+                self.browser = await self.playwright.webkit.launch(**browser_args)
+            else:
+                self.browser = await self.playwright.chromium.launch(**browser_args)
+                
+            self.default_context = self.browser
+            
+            if self.logger:
+                self.logger.debug(f"Launched {self.config.browser_type} browser", tag="BROWSER")
+                
+        except Exception as e:
+            if self.logger:
+                self.logger.error(f"Failed to launch browser: {str(e)}", tag="BROWSER")
+            raise
+            
+        return self
+
+    async def _generate_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        # Otherwise, check if we have an existing context for this config
+        config_signature = self._make_config_signature(crawlerRunConfig)
+        
+        async with self._contexts_lock:
+            if config_signature in self.contexts_by_config:
+                context = self.contexts_by_config[config_signature]
+            else:
+                # Create and setup a new context
+                context = await self.create_browser_context(crawlerRunConfig)
+                await self.setup_context(context, crawlerRunConfig)
+                self.contexts_by_config[config_signature] = context
+
+        # Create a new page from the chosen context
+        page = await context.new_page()
+        
+        return page, context
+
+    async def _get_page(self, crawlerRunConfig: CrawlerRunConfig) -> Tuple[Page, BrowserContext]:
+        """Get a page for the given configuration.
+        
+        Args:
+            crawlerRunConfig: Configuration object for the crawler run
+            
+        Returns:
+            Tuple of (Page, BrowserContext)
+        """
+        # Call parent method to ensure browser is started
+        await super().get_page(crawlerRunConfig)
+        
+        # Otherwise, check if we have an existing context for this config
+        config_signature = self._make_config_signature(crawlerRunConfig)
+        
+        async with self._contexts_lock:
+            if config_signature in self.contexts_by_config:
+                context = self.contexts_by_config[config_signature]
+            else:
+                # Create and setup a new context
+                context = await self.create_browser_context(crawlerRunConfig)
+                await self.setup_context(context, crawlerRunConfig)
+                self.contexts_by_config[config_signature] = context
+
+        # Create a new page from the chosen context
+        page = await context.new_page()
+        
+        # If a session_id is specified, store this session so we can reuse later
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            
+        return page, context
+        

crawl4ai/browser/utils.py

@@ -0,0 +1,465 @@
+"""Browser utilities module for Crawl4AI.
+
+This module provides utility functions for browser management,
+including process management, CDP connection utilities,
+and Playwright instance management.
+"""
+
+import asyncio
+import os
+import sys
+import time
+import tempfile
+import subprocess
+from typing import Optional, Tuple, Union
+import signal
+import psutil
+
+from playwright.async_api import async_playwright
+
+from ..utils import get_chromium_path
+from ..async_configs import BrowserConfig, CrawlerRunConfig
+
+from ..async_logger import AsyncLogger
+
+
+_playwright_instance = None
+
+async def get_playwright():
+    """Get or create the Playwright instance (singleton pattern).
+    
+    Returns:
+        Playwright: The Playwright instance
+    """
+    global _playwright_instance
+    if _playwright_instance is None or True:
+        _playwright_instance = await async_playwright().start()
+    return _playwright_instance
+
+async def get_browser_executable(browser_type: str) -> str:
+    """Get the path to browser executable, with platform-specific handling.
+    
+    Args:
+        browser_type: Type of browser (chromium, firefox, webkit)
+        
+    Returns:
+        Path to browser executable
+    """
+    return await get_chromium_path(browser_type)
+
+def create_temp_directory(prefix="browser-profile-") -> str:
+    """Create a temporary directory for browser data.
+    
+    Args:
+        prefix: Prefix for the temporary directory name
+        
+    Returns:
+        Path to the created temporary directory
+    """
+    return tempfile.mkdtemp(prefix=prefix)
+
+def is_windows() -> bool:
+    """Check if the current platform is Windows.
+    
+    Returns:
+        True if Windows, False otherwise
+    """
+    return sys.platform == "win32"
+
+def is_macos() -> bool:
+    """Check if the current platform is macOS.
+    
+    Returns:
+        True if macOS, False otherwise
+    """
+    return sys.platform == "darwin"
+
+def is_linux() -> bool:
+    """Check if the current platform is Linux.
+    
+    Returns:
+        True if Linux, False otherwise
+    """
+    return not (is_windows() or is_macos())
+    
+def is_browser_running(pid: Optional[int]) -> bool:
+    """Check if a process with the given PID is running.
+    
+    Args:
+        pid: Process ID to check
+        
+    Returns:
+        bool: True if the process is running, False otherwise
+    """
+    if not pid:
+        return False
+        
+    try:
+        if type(pid) is str:
+            pid = int(pid)
+        # Check if the process exists
+        if is_windows():
+            process = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], 
+                                     capture_output=True, text=True)
+            return str(pid) in process.stdout
+        else:
+            # Unix-like systems
+            os.kill(pid, 0)  # This doesn't actually kill the process, just checks if it exists
+        return True
+    except (ProcessLookupError, PermissionError, OSError):
+        return False
+
+def get_browser_disable_options() -> list:
+    """Get standard list of browser disable options for performance.
+    
+    Returns:
+        List of command-line options to disable various browser features
+    """
+    return [
+        "--disable-background-networking",
+        "--disable-background-timer-throttling",
+        "--disable-backgrounding-occluded-windows",
+        "--disable-breakpad",
+        "--disable-client-side-phishing-detection",
+        "--disable-component-extensions-with-background-pages",
+        "--disable-default-apps",
+        "--disable-extensions",
+        "--disable-features=TranslateUI",
+        "--disable-hang-monitor",
+        "--disable-ipc-flooding-protection",
+        "--disable-popup-blocking",
+        "--disable-prompt-on-repost",
+        "--disable-sync",
+        "--force-color-profile=srgb",
+        "--metrics-recording-only",
+        "--no-first-run",
+        "--password-store=basic",
+        "--use-mock-keychain",
+    ]
+
+
+async def find_optimal_browser_config(total_urls=50, verbose=True, rate_limit_delay=0.2):
+    """Find optimal browser configuration for crawling a specific number of URLs.
+    
+    Args:
+        total_urls: Number of URLs to crawl
+        verbose: Whether to print progress
+        rate_limit_delay: Delay between page loads to avoid rate limiting
+        
+    Returns:
+        dict: Contains fastest, lowest_memory, and optimal configurations
+    """
+    from .manager import BrowserManager
+    if verbose:
+        print(f"\n=== Finding optimal configuration for crawling {total_urls} URLs ===\n")
+    
+    # Generate test URLs with timestamp to avoid caching
+    timestamp = int(time.time())
+    urls = [f"https://example.com/page_{i}?t={timestamp}" for i in range(total_urls)]
+    
+    # Limit browser configurations to test (1 browser to max 10)
+    max_browsers = min(10, total_urls)
+    configs_to_test = []
+    
+    # Generate configurations (browser count, pages distribution)
+    for num_browsers in range(1, max_browsers + 1):
+        base_pages = total_urls // num_browsers
+        remainder = total_urls % num_browsers
+        
+        # Create distribution array like [3, 3, 2, 2] (some browsers get one more page)
+        if remainder > 0:
+            distribution = [base_pages + 1] * remainder + [base_pages] * (num_browsers - remainder)
+        else:
+            distribution = [base_pages] * num_browsers
+            
+        configs_to_test.append((num_browsers, distribution))
+    
+    results = []
+    
+    # Test each configuration
+    for browser_count, page_distribution in configs_to_test:
+        if verbose:
+            print(f"Testing {browser_count} browsers with distribution {tuple(page_distribution)}")
+        
+        try:
+            # Track memory if possible
+            try:
+                import psutil
+                process = psutil.Process()
+                start_memory = process.memory_info().rss / (1024 * 1024)  # MB
+            except ImportError:
+                if verbose: 
+                    print("Memory tracking not available (psutil not installed)")
+                start_memory = 0
+            
+            # Start browsers in parallel
+            managers = []
+            start_tasks = []
+            start_time = time.time()
+
+            logger = AsyncLogger(verbose=True, log_file=None)
+            
+            for i in range(browser_count):
+                config = BrowserConfig(headless=True)
+                manager = BrowserManager(browser_config=config, logger=logger)
+                start_tasks.append(manager.start())
+                managers.append(manager)
+            
+            await asyncio.gather(*start_tasks)
+            
+            # Distribute URLs among browsers
+            urls_per_manager = {}
+            url_index = 0
+            
+            for i, manager in enumerate(managers):
+                pages_for_this_browser = page_distribution[i]
+                end_index = url_index + pages_for_this_browser
+                urls_per_manager[manager] = urls[url_index:end_index]
+                url_index = end_index
+            
+            # Create pages for each browser
+            all_pages = []
+            for manager, manager_urls in urls_per_manager.items():
+                if not manager_urls:
+                    continue
+                pages = await manager.get_pages(CrawlerRunConfig(), count=len(manager_urls))
+                all_pages.extend(zip(pages, manager_urls))
+            
+            # Crawl pages with delay to avoid rate limiting
+            async def crawl_page(page_ctx, url):
+                page, _ = page_ctx
+                try:
+                    await page.goto(url)
+                    if rate_limit_delay > 0:
+                        await asyncio.sleep(rate_limit_delay)
+                    title = await page.title()
+                    return title
+                finally:
+                    await page.close()
+            
+            crawl_start = time.time()
+            crawl_tasks = [crawl_page(page_ctx, url) for page_ctx, url in all_pages]
+            await asyncio.gather(*crawl_tasks)
+            crawl_time = time.time() - crawl_start
+            total_time = time.time() - start_time
+            
+            # Measure final memory usage
+            if start_memory > 0:
+                end_memory = process.memory_info().rss / (1024 * 1024)
+                memory_used = end_memory - start_memory
+            else:
+                memory_used = 0
+            
+            # Close all browsers
+            for manager in managers:
+                await manager.close()
+            
+            # Calculate metrics
+            pages_per_second = total_urls / crawl_time
+            
+            # Calculate efficiency score (higher is better)
+            # This balances speed vs memory
+            if memory_used > 0:
+                efficiency = pages_per_second / (memory_used + 1)
+            else:
+                efficiency = pages_per_second
+            
+            # Store result
+            result = {
+                "browser_count": browser_count,
+                "distribution": tuple(page_distribution),
+                "crawl_time": crawl_time,
+                "total_time": total_time,
+                "memory_used": memory_used,
+                "pages_per_second": pages_per_second, 
+                "efficiency": efficiency
+            }
+            
+            results.append(result)
+            
+            if verbose:
+                print(f"  ✓ Crawled {total_urls} pages in {crawl_time:.2f}s ({pages_per_second:.1f} pages/sec)")
+                if memory_used > 0:
+                    print(f"  ✓ Memory used: {memory_used:.1f}MB ({memory_used/total_urls:.1f}MB per page)")
+                print(f"  ✓ Efficiency score: {efficiency:.4f}")
+            
+        except Exception as e:
+            if verbose:
+                print(f"  ✗ Error: {str(e)}")
+            
+            # Clean up
+            for manager in managers:
+                try:
+                    await manager.close()
+                except:
+                    pass
+    
+    # If no successful results, return None
+    if not results:
+        return None
+    
+    # Find best configurations
+    fastest = sorted(results, key=lambda x: x["crawl_time"])[0]
+    
+    # Only consider memory if available
+    memory_results = [r for r in results if r["memory_used"] > 0]
+    if memory_results:
+        lowest_memory = sorted(memory_results, key=lambda x: x["memory_used"])[0]
+    else:
+        lowest_memory = fastest
+    
+    # Find most efficient (balanced speed vs memory)
+    optimal = sorted(results, key=lambda x: x["efficiency"], reverse=True)[0]
+    
+    # Print summary
+    if verbose:
+        print("\n=== OPTIMAL CONFIGURATIONS ===")
+        print(f"⚡ Fastest: {fastest['browser_count']} browsers {fastest['distribution']}")
+        print(f"   {fastest['crawl_time']:.2f}s, {fastest['pages_per_second']:.1f} pages/sec")
+        
+        print(f"💾 Memory-efficient: {lowest_memory['browser_count']} browsers {lowest_memory['distribution']}")
+        if lowest_memory["memory_used"] > 0:
+            print(f"   {lowest_memory['memory_used']:.1f}MB, {lowest_memory['memory_used']/total_urls:.2f}MB per page")
+        
+        print(f"🌟 Balanced optimal: {optimal['browser_count']} browsers {optimal['distribution']}")
+        print(f"   {optimal['crawl_time']:.2f}s, {optimal['pages_per_second']:.1f} pages/sec, score: {optimal['efficiency']:.4f}")
+    
+    return {
+        "fastest": fastest,
+        "lowest_memory": lowest_memory,
+        "optimal": optimal,
+        "all_configs": results
+    }
+
+
+# Find process ID of the existing browser using os
+def find_process_by_port(port: int) -> str:
+    """Find process ID listening on a specific port.
+    
+    Args:
+        port: Port number to check
+        
+    Returns:
+        str: Process ID or empty string if not found
+    """
+    try:
+        if is_windows():
+            cmd = f"netstat -ano | findstr :{port}"
+            result = subprocess.check_output(cmd, shell=True).decode()
+            return result.strip().split()[-1] if result else ""
+        else:
+            cmd = f"lsof -i :{port} -t"
+            return subprocess.check_output(cmd, shell=True).decode().strip()
+    except subprocess.CalledProcessError:
+        return ""
+    
+async def check_process_is_running(process: subprocess.Popen, delay: float = 0.5) -> Tuple[bool, Optional[int], bytes, bytes]:
+    """Perform a quick check to make sure the browser started successfully."""
+    if not process:
+        return False, None, b"", b""
+        
+    # Check that process started without immediate termination
+    await asyncio.sleep(delay)
+    if process.poll() is not None:
+        # Process already terminated
+        stdout, stderr = b"", b""
+        try:
+            stdout, stderr = process.communicate(timeout=0.5)
+        except subprocess.TimeoutExpired:
+            pass
+
+        return False, process.returncode, stdout, stderr
+            
+
+    return True, 0, b"", b""
+
+
+def terminate_process(
+    pid: Union[int, str], 
+    timeout: float = 5.0,
+    force_kill_timeout: float = 3.0,
+    logger = None
+) -> Tuple[bool, Optional[str]]:
+    """
+    Robustly terminate a process across platforms with verification.
+    
+    Args:
+        pid: Process ID to terminate (int or string)
+        timeout: Seconds to wait for graceful termination before force killing
+        force_kill_timeout: Seconds to wait after force kill before considering it failed
+        logger: Optional logger object with error, warning, and info methods
+        
+    Returns:
+        Tuple of (success: bool, error_message: Optional[str])
+    """
+    # Convert pid to int if it's a string
+    if isinstance(pid, str):
+        try:
+            pid = int(pid)
+        except ValueError:
+            error_msg = f"Invalid PID format: {pid}"
+            if logger:
+                logger.error(error_msg)
+            return False, error_msg
+    
+    # Check if process exists
+    if not psutil.pid_exists(pid):
+        return True, None  # Process already terminated
+    
+    try:
+        process = psutil.Process(pid)
+        
+        # First attempt: graceful termination
+        if logger:
+            logger.info(f"Attempting graceful termination of process {pid}")
+            
+        if os.name == 'nt':  # Windows
+            subprocess.run(["taskkill", "/PID", str(pid)], 
+                          stdout=subprocess.DEVNULL, 
+                          stderr=subprocess.DEVNULL, 
+                          check=False)
+        else:  # Unix/Linux/MacOS
+            process.send_signal(signal.SIGTERM)
+        
+        # Wait for process to terminate
+        try:
+            process.wait(timeout=timeout)
+            if logger:
+                logger.info(f"Process {pid} terminated gracefully")
+            return True, None
+        except psutil.TimeoutExpired:
+            if logger:
+                logger.warning(f"Process {pid} did not terminate gracefully within {timeout} seconds, forcing termination")
+        
+        # Second attempt: force kill
+        if os.name == 'nt':  # Windows
+            subprocess.run(["taskkill", "/F", "/PID", str(pid)], 
+                          stdout=subprocess.DEVNULL, 
+                          stderr=subprocess.DEVNULL, 
+                          check=False)
+        else:  # Unix/Linux/MacOS
+            process.send_signal(signal.SIGKILL)
+        
+        # Verify process is killed
+        gone, alive = psutil.wait_procs([process], timeout=force_kill_timeout)
+        if process in alive:
+            error_msg = f"Failed to kill process {pid} even after force kill"
+            if logger:
+                logger.error(error_msg)
+            return False, error_msg
+            
+        if logger:
+            logger.info(f"Process {pid} terminated by force")
+        return True, None
+        
+    except psutil.NoSuchProcess:
+        # Process terminated while we were working with it
+        if logger:
+            logger.info(f"Process {pid} already terminated")
+        return True, None
+        
+    except Exception as e:
+        error_msg = f"Error terminating process {pid}: {str(e)}"
+        if logger:
+            logger.error(error_msg)
+        return False, error_msg

crawl4ai/browser_manager.py

@@ -0,0 +1,932 @@
+import asyncio
+import time
+from typing import List, Optional
+import os
+import sys
+import shutil
+import tempfile
+import subprocess
+from playwright.async_api import BrowserContext
+import hashlib
+from .js_snippet import load_js_script
+from .config import DOWNLOAD_PAGE_TIMEOUT
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from playwright_stealth import StealthConfig
+from .utils import get_chromium_path
+
+stealth_config = StealthConfig(
+    webdriver=True,
+    chrome_app=True,
+    chrome_csi=True,
+    chrome_load_times=True,
+    chrome_runtime=True,
+    navigator_languages=True,
+    navigator_plugins=True,
+    navigator_permissions=True,
+    webgl_vendor=True,
+    outerdimensions=True,
+    navigator_hardware_concurrency=True,
+    media_codecs=True,
+)
+
+BROWSER_DISABLE_OPTIONS = [
+    "--disable-background-networking",
+    "--disable-background-timer-throttling",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-breakpad",
+    "--disable-client-side-phishing-detection",
+    "--disable-component-extensions-with-background-pages",
+    "--disable-default-apps",
+    "--disable-extensions",
+    "--disable-features=TranslateUI",
+    "--disable-hang-monitor",
+    "--disable-ipc-flooding-protection",
+    "--disable-popup-blocking",
+    "--disable-prompt-on-repost",
+    "--disable-sync",
+    "--force-color-profile=srgb",
+    "--metrics-recording-only",
+    "--no-first-run",
+    "--password-store=basic",
+    "--use-mock-keychain",
+]
+
+
+class ManagedBrowser:
+    """
+    Manages the browser process and context. This class allows to connect to the browser using CDP protocol.
+
+    Attributes:
+        browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                            Default: "chromium".
+        user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                     temporary directory may be used. Default: None.
+        headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                         Default: True.
+        browser_process (subprocess.Popen): The process object for the browser.
+        temp_dir (str): Temporary directory for user data if not provided.
+        debugging_port (int): Port for debugging the browser.
+        host (str): Host for debugging the browser.
+
+        Methods:
+            start(): Starts the browser process and returns the CDP endpoint URL.
+            _get_browser_path(): Returns the browser executable path based on OS and browser type.
+            _get_browser_args(): Returns browser-specific command line arguments.
+            _get_user_data_dir(): Returns the user data directory path.
+            _cleanup(): Terminates the browser process and removes the temporary directory.
+            create_profile(): Static method to create a user profile by launching a browser for user interaction.
+    """
+
+    browser_type: str
+    user_data_dir: str
+    headless: bool
+    browser_process: subprocess.Popen
+    temp_dir: str
+    debugging_port: int
+    host: str
+
+    def __init__(
+        self,
+        browser_type: str = "chromium",
+        user_data_dir: Optional[str] = None,
+        headless: bool = False,
+        logger=None,
+        host: str = "localhost",
+        debugging_port: int = 9222,
+        cdp_url: Optional[str] = None, 
+    ):
+        """
+        Initialize the ManagedBrowser instance.
+
+        Args:
+            browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                                Default: "chromium".
+            user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                         temporary directory may be used. Default: None.
+            headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                             Default: True.
+            logger (logging.Logger): Logger instance for logging messages. Default: None.
+            host (str): Host for debugging the browser. Default: "localhost".
+            debugging_port (int): Port for debugging the browser. Default: 9222.
+            cdp_url (str or None): CDP URL to connect to the browser. Default: None.
+        """
+        self.browser_type = browser_type
+        self.user_data_dir = user_data_dir
+        self.headless = headless
+        self.browser_process = None
+        self.temp_dir = None
+        self.debugging_port = debugging_port
+        self.host = host
+        self.logger = logger
+        self.shutting_down = False
+        self.cdp_url = cdp_url
+
+    async def start(self) -> str:
+        """
+        Starts the browser process or returns CDP endpoint URL.
+        If cdp_url is provided, returns it directly.
+        If user_data_dir is not provided for local browser, creates a temporary directory.
+        
+        Returns:
+            str: CDP endpoint URL
+        """
+        # If CDP URL provided, just return it
+        if self.cdp_url:
+            return self.cdp_url
+
+        # Create temp dir if needed
+        if not self.user_data_dir:
+            self.temp_dir = tempfile.mkdtemp(prefix="browser-profile-")
+            self.user_data_dir = self.temp_dir
+
+        # Get browser path and args based on OS and browser type
+        # browser_path = self._get_browser_path()
+        args = await self._get_browser_args()
+
+        # Start browser process
+        try:
+            # Use DETACHED_PROCESS flag on Windows to fully detach the process
+            # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group
+            if sys.platform == "win32":
+                self.browser_process = subprocess.Popen(
+                    args, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
+                )
+            else:
+                self.browser_process = subprocess.Popen(
+                    args, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    preexec_fn=os.setpgrp  # Start in a new process group
+                )
+                
+            # We'll monitor for a short time to make sure it starts properly, but won't keep monitoring
+            await asyncio.sleep(0.5)  # Give browser time to start
+            await self._initial_startup_check()
+            await asyncio.sleep(2)  # Give browser time to start
+            return f"http://{self.host}:{self.debugging_port}"
+        except Exception as e:
+            await self.cleanup()
+            raise Exception(f"Failed to start browser: {e}")
+
+    async def _initial_startup_check(self):
+        """
+        Perform a quick check to make sure the browser started successfully.
+        This only runs once at startup rather than continuously monitoring.
+        """
+        if not self.browser_process:
+            return
+            
+        # Check that process started without immediate termination
+        await asyncio.sleep(0.5)
+        if self.browser_process.poll() is not None:
+            # Process already terminated
+            stdout, stderr = b"", b""
+            try:
+                stdout, stderr = self.browser_process.communicate(timeout=0.5)
+            except subprocess.TimeoutExpired:
+                pass
+                
+            self.logger.error(
+                message="Browser process terminated during startup | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
+                tag="ERROR",
+                params={
+                    "code": self.browser_process.returncode,
+                    "stdout": stdout.decode() if stdout else "",
+                    "stderr": stderr.decode() if stderr else "",
+                },
+            )
+    
+    async def _monitor_browser_process(self):
+        """
+        Monitor the browser process for unexpected termination.
+
+        How it works:
+        1. Read stdout and stderr from the browser process.
+        2. If the process has terminated, log the error message and terminate the browser.
+        3. If the shutting_down flag is set, log the normal termination message.
+        4. If any other error occurs, log the error message.
+
+        Note: This method should be called in a separate task to avoid blocking the main event loop.
+        This is DEPRECATED and should not be used for builtin browsers that need to outlive the Python process.
+        """
+        if self.browser_process:
+            try:
+                stdout, stderr = await asyncio.gather(
+                    asyncio.to_thread(self.browser_process.stdout.read),
+                    asyncio.to_thread(self.browser_process.stderr.read),
+                )
+
+                # Check shutting_down flag BEFORE logging anything
+                if self.browser_process.poll() is not None:
+                    if not self.shutting_down:
+                        self.logger.error(
+                            message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
+                            tag="ERROR",
+                            params={
+                                "code": self.browser_process.returncode,
+                                "stdout": stdout.decode(),
+                                "stderr": stderr.decode(),
+                            },
+                        )
+                        await self.cleanup()
+                    else:
+                        self.logger.info(
+                            message="Browser process terminated normally | Code: {code}",
+                            tag="INFO",
+                            params={"code": self.browser_process.returncode},
+                        )
+            except Exception as e:
+                if not self.shutting_down:
+                    self.logger.error(
+                        message="Error monitoring browser process: {error}",
+                        tag="ERROR",
+                        params={"error": str(e)},
+                    )
+
+    def _get_browser_path_WIP(self) -> str:
+        """Returns the browser executable path based on OS and browser type"""
+        if sys.platform == "darwin":  # macOS
+            paths = {
+                "chromium": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                "firefox": "/Applications/Firefox.app/Contents/MacOS/firefox",
+                "webkit": "/Applications/Safari.app/Contents/MacOS/Safari",
+            }
+        elif sys.platform == "win32":  # Windows
+            paths = {
+                "chromium": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+                "firefox": "C:\\Program Files\\Mozilla Firefox\\firefox.exe",
+                "webkit": None,  # WebKit not supported on Windows
+            }
+        else:  # Linux
+            paths = {
+                "chromium": "google-chrome",
+                "firefox": "firefox",
+                "webkit": None,  # WebKit not supported on Linux
+            }
+
+        return paths.get(self.browser_type)
+
+    async def _get_browser_path(self) -> str:
+        browser_path = await get_chromium_path(self.browser_type)
+        return browser_path
+
+    async def _get_browser_args(self) -> List[str]:
+        """Returns browser-specific command line arguments"""
+        base_args = [await self._get_browser_path()]
+
+        if self.browser_type == "chromium":
+            args = [
+                f"--remote-debugging-port={self.debugging_port}",
+                f"--user-data-dir={self.user_data_dir}",
+            ]
+            if self.headless:
+                args.append("--headless=new")
+        elif self.browser_type == "firefox":
+            args = [
+                "--remote-debugging-port",
+                str(self.debugging_port),
+                "--profile",
+                self.user_data_dir,
+            ]
+            if self.headless:
+                args.append("--headless")
+        else:
+            raise NotImplementedError(f"Browser type {self.browser_type} not supported")
+
+        return base_args + args
+
+    async def cleanup(self):
+        """Cleanup browser process and temporary directory"""
+        # Set shutting_down flag BEFORE any termination actions
+        self.shutting_down = True
+
+        if self.browser_process:
+            try:
+                # For builtin browsers that should persist, we should check if it's a detached process
+                # Only terminate if we have proper control over the process
+                if not self.browser_process.poll():
+                    # Process is still running
+                    self.browser_process.terminate()
+                    # Wait for process to end gracefully
+                    for _ in range(10):  # 10 attempts, 100ms each
+                        if self.browser_process.poll() is not None:
+                            break
+                        await asyncio.sleep(0.1)
+
+                    # Force kill if still running
+                    if self.browser_process.poll() is None:
+                        if sys.platform == "win32":
+                            # On Windows we might need taskkill for detached processes
+                            try:
+                                subprocess.run(["taskkill", "/F", "/PID", str(self.browser_process.pid)])
+                            except Exception:
+                                self.browser_process.kill()
+                        else:
+                            self.browser_process.kill()
+                        await asyncio.sleep(0.1)  # Brief wait for kill to take effect
+
+            except Exception as e:
+                self.logger.error(
+                    message="Error terminating browser: {error}",
+                    tag="ERROR", 
+                    params={"error": str(e)},
+                )
+
+        if self.temp_dir and os.path.exists(self.temp_dir):
+            try:
+                shutil.rmtree(self.temp_dir)
+            except Exception as e:
+                self.logger.error(
+                    message="Error removing temporary directory: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)},
+                )
+                
+    # These methods have been moved to BrowserProfiler class
+    @staticmethod
+    async def create_profile(browser_config=None, profile_name=None, logger=None):
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Creates a browser profile by launching a browser for interactive user setup
+        and waits until the user closes it. The profile is stored in a directory that
+        can be used later with BrowserConfig.user_data_dir.
+        
+        Please use BrowserProfiler.create_profile() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            profile_path = await profiler.create_profile(profile_name="my-login-profile")
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler(logger=logger)
+        return await profiler.create_profile(profile_name=profile_name, browser_config=browser_config)
+    
+    @staticmethod
+    def list_profiles():
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Lists all available browser profiles in the Crawl4AI profiles directory.
+        
+        Please use BrowserProfiler.list_profiles() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            profiles = profiler.list_profiles()
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler()
+        return profiler.list_profiles()
+        
+    @staticmethod
+    def delete_profile(profile_name_or_path):
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Delete a browser profile by name or path.
+        
+        Please use BrowserProfiler.delete_profile() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            success = profiler.delete_profile("my-profile")
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler()
+        return profiler.delete_profile(profile_name_or_path)
+
+
+
+
+class BrowserManager:
+    """
+    Manages the browser instance and context.
+
+    Attributes:
+        config (BrowserConfig): Configuration object containing all browser settings
+        logger: Logger instance for recording events and errors
+        browser (Browser): The browser instance
+        default_context (BrowserContext): The default browser context
+        managed_browser (ManagedBrowser): The managed browser instance
+        playwright (Playwright): The Playwright instance
+        sessions (dict): Dictionary to store session information
+        session_ttl (int): Session timeout in seconds
+    """
+
+    _playwright_instance = None
+    
+    @classmethod
+    async def get_playwright(cls):
+        from playwright.async_api import async_playwright
+        if cls._playwright_instance is None:
+            cls._playwright_instance = await async_playwright().start()
+        return cls._playwright_instance    
+
+    def __init__(self, browser_config: BrowserConfig, logger=None):
+        """
+        Initialize the BrowserManager with a browser configuration.
+
+        Args:
+            browser_config (BrowserConfig): Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+        """
+        self.config: BrowserConfig = browser_config
+        self.logger = logger
+
+        # Browser state
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+
+        # Session management
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+
+        # Keep track of contexts by a "config signature," so each unique config reuses a single context
+        self.contexts_by_config = {}
+        self._contexts_lock = asyncio.Lock() 
+
+        # Initialize ManagedBrowser if needed
+        if self.config.use_managed_browser:
+            self.managed_browser = ManagedBrowser(
+                browser_type=self.config.browser_type,
+                user_data_dir=self.config.user_data_dir,
+                headless=self.config.headless,
+                logger=self.logger,
+                debugging_port=self.config.debugging_port,
+                cdp_url=self.config.cdp_url,
+            )
+
+    async def start(self):
+        """
+        Start the browser instance and set up the default context.
+
+        How it works:
+        1. Check if Playwright is already initialized.
+        2. If not, initialize Playwright.
+        3. If managed browser is used, start it and connect to the CDP endpoint.
+        4. If managed browser is not used, launch the browser and set up the default context.
+
+        Note: This method should be called in a separate task to avoid blocking the main event loop.
+        """
+        self.playwright  = await self.get_playwright()
+        if self.playwright is None:
+            from playwright.async_api import async_playwright
+
+            self.playwright = await async_playwright().start()
+
+        if self.config.cdp_url or self.config.use_managed_browser:
+            self.config.use_managed_browser = True
+            cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url
+            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+            else:
+                self.default_context = await self.create_browser_context()
+            await self.setup_context(self.default_context)
+        else:
+            browser_args = self._build_browser_args()
+
+            # Launch appropriate browser type
+            if self.config.browser_type == "firefox":
+                self.browser = await self.playwright.firefox.launch(**browser_args)
+            elif self.config.browser_type == "webkit":
+                self.browser = await self.playwright.webkit.launch(**browser_args)
+            else:
+                self.browser = await self.playwright.chromium.launch(**browser_args)
+
+            self.default_context = self.browser
+
+
+    def _build_browser_args(self) -> dict:
+        """Build browser launch arguments from config."""
+        args = [
+            "--disable-gpu",
+            "--disable-gpu-compositing",
+            "--disable-software-rasterizer",
+            "--no-sandbox",
+            "--disable-dev-shm-usage",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--disable-infobars",
+            "--window-position=0,0",
+            "--ignore-certificate-errors",
+            "--ignore-certificate-errors-spki-list",
+            "--disable-blink-features=AutomationControlled",
+            "--window-position=400,0",
+            "--disable-renderer-backgrounding",
+            "--disable-ipc-flooding-protection",
+            "--force-color-profile=srgb",
+            "--mute-audio",
+            "--disable-background-timer-throttling",
+            # "--single-process",
+            f"--window-size={self.config.viewport_width},{self.config.viewport_height}",
+        ]
+
+        if self.config.light_mode:
+            args.extend(BROWSER_DISABLE_OPTIONS)
+
+        if self.config.text_mode:
+            args.extend(
+                [
+                    "--blink-settings=imagesEnabled=false",
+                    "--disable-remote-fonts",
+                    "--disable-images",
+                    "--disable-javascript",
+                    "--disable-software-rasterizer",
+                    "--disable-dev-shm-usage",
+                ]
+            )
+
+        if self.config.extra_args:
+            args.extend(self.config.extra_args)
+
+        browser_args = {"headless": self.config.headless, "args": args}
+
+        if self.config.chrome_channel:
+            browser_args["channel"] = self.config.chrome_channel
+
+        if self.config.accept_downloads:
+            browser_args["downloads_path"] = self.config.downloads_path or os.path.join(
+                os.getcwd(), "downloads"
+            )
+            os.makedirs(browser_args["downloads_path"], exist_ok=True)
+
+        if self.config.proxy or self.config.proxy_config:
+            from playwright.async_api import ProxySettings
+
+            proxy_settings = (
+                ProxySettings(server=self.config.proxy)
+                if self.config.proxy
+                else ProxySettings(
+                    server=self.config.proxy_config.server,
+                    username=self.config.proxy_config.username,
+                    password=self.config.proxy_config.password,
+                )
+            )
+            browser_args["proxy"] = proxy_settings
+
+        return browser_args
+
+    async def setup_context(
+        self,
+        context: BrowserContext,
+        crawlerRunConfig: CrawlerRunConfig = None,
+        is_default=False,
+    ):
+        """
+        Set up a browser context with the configured options.
+
+        How it works:
+        1. Set extra HTTP headers if provided.
+        2. Add cookies if provided.
+        3. Load storage state if provided.
+        4. Accept downloads if enabled.
+        5. Set default timeouts for navigation and download.
+        6. Set user agent if provided.
+        7. Set browser hints if provided.
+        8. Set proxy if provided.
+        9. Set downloads path if provided.
+        10. Set storage state if provided.
+        11. Set cache if provided.
+        12. Set extra HTTP headers if provided.
+        13. Add cookies if provided.
+        14. Set default timeouts for navigation and download if enabled.
+        15. Set user agent if provided.
+        16. Set browser hints if provided.
+
+        Args:
+            context (BrowserContext): The browser context to set up
+            crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings
+            is_default (bool): Flag indicating if this is the default context
+        Returns:
+            None
+        """
+        if self.config.headers:
+            await context.set_extra_http_headers(self.config.headers)
+
+        if self.config.cookies:
+            await context.add_cookies(self.config.cookies)
+
+        if self.config.storage_state:
+            await context.storage_state(path=None)
+
+        if self.config.accept_downloads:
+            context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            if self.config.downloads_path:
+                context._impl_obj._options["accept_downloads"] = True
+                context._impl_obj._options[
+                    "downloads_path"
+                ] = self.config.downloads_path
+
+        # Handle user agent and browser hints
+        if self.config.user_agent:
+            combined_headers = {
+                "User-Agent": self.config.user_agent,
+                "sec-ch-ua": self.config.browser_hint,
+            }
+            combined_headers.update(self.config.headers)
+            await context.set_extra_http_headers(combined_headers)
+
+        # Add default cookie
+        await context.add_cookies(
+            [
+                {
+                    "name": "cookiesEnabled",
+                    "value": "true",
+                    "url": crawlerRunConfig.url
+                    if crawlerRunConfig
+                    else "https://crawl4ai.com/",
+                }
+            ]
+        )
+
+        # Handle navigator overrides
+        if crawlerRunConfig:
+            if (
+                crawlerRunConfig.override_navigator
+                or crawlerRunConfig.simulate_user
+                or crawlerRunConfig.magic
+            ):
+                await context.add_init_script(load_js_script("navigator_overrider"))        
+
+    async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None):
+        """
+        Creates and returns a new browser context with configured settings.
+        Applies text-only mode settings if text_mode is enabled in config.
+
+        Returns:
+            Context: Browser context object with the specified configurations
+        """
+        # Base settings
+        user_agent = self.config.headers.get("User-Agent", self.config.user_agent) 
+        viewport_settings = {
+            "width": self.config.viewport_width,
+            "height": self.config.viewport_height,
+        }
+        proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
+
+        blocked_extensions = [
+            # Images
+            "jpg",
+            "jpeg",
+            "png",
+            "gif",
+            "webp",
+            "svg",
+            "ico",
+            "bmp",
+            "tiff",
+            "psd",
+            # Fonts
+            "woff",
+            "woff2",
+            "ttf",
+            "otf",
+            "eot",
+            # Styles
+            # 'css', 'less', 'scss', 'sass',
+            # Media
+            "mp4",
+            "webm",
+            "ogg",
+            "avi",
+            "mov",
+            "wmv",
+            "flv",
+            "m4v",
+            "mp3",
+            "wav",
+            "aac",
+            "m4a",
+            "opus",
+            "flac",
+            # Documents
+            "pdf",
+            "doc",
+            "docx",
+            "xls",
+            "xlsx",
+            "ppt",
+            "pptx",
+            # Archives
+            "zip",
+            "rar",
+            "7z",
+            "tar",
+            "gz",
+            # Scripts and data
+            "xml",
+            "swf",
+            "wasm",
+        ]
+
+        # Common context settings
+        context_settings = {
+            "user_agent": user_agent,
+            "viewport": viewport_settings,
+            "proxy": proxy_settings,
+            "accept_downloads": self.config.accept_downloads,
+            "storage_state": self.config.storage_state,
+            "ignore_https_errors": self.config.ignore_https_errors,
+            "device_scale_factor": 1.0,
+            "java_script_enabled": self.config.java_script_enabled,
+        }
+        
+        if crawlerRunConfig:
+            # Check if there is value for crawlerRunConfig.proxy_config set add that to context
+            if crawlerRunConfig.proxy_config:
+                proxy_settings = {
+                    "server": crawlerRunConfig.proxy_config.server,
+                }
+                if crawlerRunConfig.proxy_config.username:
+                    proxy_settings.update({
+                        "username": crawlerRunConfig.proxy_config.username,
+                        "password": crawlerRunConfig.proxy_config.password,
+                    })
+                context_settings["proxy"] = proxy_settings
+
+        if self.config.text_mode:
+            text_mode_settings = {
+                "has_touch": False,
+                "is_mobile": False,
+            }
+            # Update context settings with text mode settings
+            context_settings.update(text_mode_settings)
+
+        # Create and return the context with all settings
+        context = await self.browser.new_context(**context_settings)
+
+        # Apply text mode settings if enabled
+        if self.config.text_mode:
+            # Create and apply route patterns for each extension
+            for ext in blocked_extensions:
+                await context.route(f"**/*.{ext}", lambda route: route.abort())
+        return context
+
+    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
+        """
+        Converts the crawlerRunConfig into a dict, excludes ephemeral fields,
+        then returns a hash of the sorted JSON. This yields a stable signature
+        that identifies configurations requiring a unique browser context.
+        """
+        import json
+
+        config_dict = crawlerRunConfig.__dict__.copy()
+        # Exclude items that do not affect browser-level setup.
+        # Expand or adjust as needed, e.g. chunking_strategy is purely for data extraction, not for browser config.
+        ephemeral_keys = [
+            "session_id",
+            "js_code",
+            "scraping_strategy",
+            "extraction_strategy",
+            "chunking_strategy",
+            "cache_mode",
+            "content_filter",
+            "semaphore_count",
+            "url"
+        ]
+        for key in ephemeral_keys:
+            if key in config_dict:
+                del config_dict[key]
+        # Convert to canonical JSON string
+        signature_json = json.dumps(config_dict, sort_keys=True, default=str)
+
+        # Hash the JSON so we get a compact, unique string
+        signature_hash = hashlib.sha256(signature_json.encode("utf-8")).hexdigest()
+        return signature_hash
+
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig):
+        """
+        Get a page for the given session ID, creating a new one if needed.
+
+        Args:
+            crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings
+
+        Returns:
+            (page, context): The Page and its BrowserContext
+        """
+        self._cleanup_expired_sessions()
+
+        # If a session_id is provided and we already have it, reuse that page + context
+        if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions:
+            context, page, _ = self.sessions[crawlerRunConfig.session_id]
+            # Update last-used timestamp
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            return page, context
+
+        # If using a managed browser, just grab the shared default_context
+        if self.config.use_managed_browser:
+            context = self.default_context
+            pages = context.pages
+            page = next((p for p in pages if p.url == crawlerRunConfig.url), None)
+            if not page:
+                page = await context.new_page()
+        else:
+            # Otherwise, check if we have an existing context for this config
+            config_signature = self._make_config_signature(crawlerRunConfig)
+
+            async with self._contexts_lock:
+                if config_signature in self.contexts_by_config:
+                    context = self.contexts_by_config[config_signature]
+                else:
+                    # Create and setup a new context
+                    context = await self.create_browser_context(crawlerRunConfig)
+                    await self.setup_context(context, crawlerRunConfig)
+                    self.contexts_by_config[config_signature] = context
+
+            # Create a new page from the chosen context
+            page = await context.new_page()
+
+        # If a session_id is specified, store this session so we can reuse later
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+
+        return page, context
+
+    async def kill_session(self, session_id: str):
+        """
+        Kill a browser session and clean up resources.
+
+        Args:
+            session_id (str): The session ID to kill.
+        """
+        if session_id in self.sessions:
+            context, page, _ = self.sessions[session_id]
+            await page.close()
+            if not self.config.use_managed_browser:
+                await context.close()
+            del self.sessions[session_id]
+
+    def _cleanup_expired_sessions(self):
+        """Clean up expired sessions based on TTL."""
+        current_time = time.time()
+        expired_sessions = [
+            sid
+            for sid, (_, _, last_used) in self.sessions.items()
+            if current_time - last_used > self.session_ttl
+        ]
+        for sid in expired_sessions:
+            asyncio.create_task(self.kill_session(sid))
+
+    async def close(self):
+        """Close all browser resources and clean up."""
+        if self.config.cdp_url:
+            return
+        
+        if self.config.sleep_on_close:
+            await asyncio.sleep(0.5)
+
+        session_ids = list(self.sessions.keys())
+        for session_id in session_ids:
+            await self.kill_session(session_id)
+
+        # Now close all contexts we created. This reclaims memory from ephemeral contexts.
+        for ctx in self.contexts_by_config.values():
+            try:
+                await ctx.close()
+            except Exception as e:
+                self.logger.error(
+                    message="Error closing context: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)}
+                )
+        self.contexts_by_config.clear()
+
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+
+        if self.managed_browser:
+            await asyncio.sleep(0.5)
+            await self.managed_browser.cleanup()
+            self.managed_browser = None
+
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None

crawl4ai/browser_profiler.py

@@ -0,0 +1,974 @@
+"""
+Browser Profiler Module
+
+This module provides a dedicated class for managing browser profiles
+that can be used for identity-based crawling with Crawl4AI.
+"""
+
+import os
+import asyncio
+import signal
+import sys
+import datetime
+import uuid
+import shutil
+import json
+import subprocess
+import time
+from typing import List, Dict, Optional, Any, Tuple
+from colorama import Fore, Style, init
+
+from .async_configs import BrowserConfig
+from .browser_manager import ManagedBrowser
+from .async_logger import AsyncLogger, AsyncLoggerBase
+from .utils import get_home_folder
+
+
+class BrowserProfiler:
+    """
+    A dedicated class for managing browser profiles for Crawl4AI.
+    
+    The BrowserProfiler allows you to:
+    - Create browser profiles interactively
+    - List available profiles
+    - Delete profiles when no longer needed
+    - Get profile paths for use in BrowserConfig
+    
+    Profiles are stored by default in ~/.crawl4ai/profiles/
+    """
+    
+    def __init__(self, logger: Optional[AsyncLoggerBase] = None):
+        """
+        Initialize the BrowserProfiler.
+        
+        Args:
+            logger (AsyncLoggerBase, optional): Logger for outputting messages.
+                If None, a default AsyncLogger will be created.
+        """
+        # Initialize colorama for colorful terminal output
+        init()
+        
+        # Create a logger if not provided
+        if logger is None:
+            self.logger = AsyncLogger(verbose=True)
+        elif not isinstance(logger, AsyncLoggerBase):
+            self.logger = AsyncLogger(verbose=True)
+        else:
+            self.logger = logger
+            
+        # Ensure profiles directory exists
+        self.profiles_dir = os.path.join(get_home_folder(), "profiles")
+        os.makedirs(self.profiles_dir, exist_ok=True)
+        
+        # Builtin browser config file
+        self.builtin_browser_dir = os.path.join(get_home_folder(), "builtin-browser")
+        self.builtin_config_file = os.path.join(self.builtin_browser_dir, "browser_config.json")
+        os.makedirs(self.builtin_browser_dir, exist_ok=True)
+    
+    async def create_profile(self, 
+                            profile_name: Optional[str] = None, 
+                            browser_config: Optional[BrowserConfig] = None) -> Optional[str]:
+        """
+        Creates a browser profile by launching a browser for interactive user setup
+        and waits until the user closes it. The profile is stored in a directory that
+        can be used later with BrowserConfig.user_data_dir.
+        
+        Args:
+            profile_name (str, optional): Name for the profile directory.
+                If None, a name is generated based on timestamp.
+            browser_config (BrowserConfig, optional): Configuration for the browser.
+                If None, a default configuration is used with headless=False.
+                
+        Returns:
+            str: Path to the created profile directory, or None if creation failed
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Create a profile interactively
+            profile_path = await profiler.create_profile(
+                profile_name="my-login-profile"
+            )
+            
+            # Use the profile in a crawler
+            browser_config = BrowserConfig(
+                headless=True,
+                use_managed_browser=True,
+                user_data_dir=profile_path
+            )
+            
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                # The crawler will now use your profile with all your cookies and login state
+                result = await crawler.arun("https://example.com/dashboard")
+            ```
+        """
+        # Create default browser config if none provided
+        if browser_config is None:
+            from .async_configs import BrowserConfig
+            browser_config = BrowserConfig(
+                browser_type="chromium",
+                headless=False,  # Must be visible for user interaction
+                verbose=True
+            )
+        else:
+            # Ensure headless is False for user interaction
+            browser_config.headless = False
+            
+        # Generate profile name if not provided
+        if not profile_name:
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"profile_{timestamp}_{uuid.uuid4().hex[:6]}"
+            
+        # Sanitize profile name (replace spaces and special chars)
+        profile_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in profile_name)
+        
+        # Set user data directory
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        os.makedirs(profile_path, exist_ok=True)
+        
+        # Print instructions for the user with colorama formatting
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="PROFILE")
+        self.logger.info(f"Creating browser profile: {Fore.GREEN}{profile_name}{Style.RESET_ALL}", tag="PROFILE")
+        self.logger.info(f"Profile directory: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+        
+        self.logger.info("\nInstructions:", tag="PROFILE")
+        self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE")
+        self.logger.info(f"2. {Fore.CYAN}Log in to websites{Style.RESET_ALL}, configure settings, etc. as needed.", tag="PROFILE")
+        self.logger.info(f"3. When you're done, {Fore.YELLOW}press 'q' in this terminal{Style.RESET_ALL} to close the browser.", tag="PROFILE")
+        self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE")
+        self.logger.info(f"{border}\n", tag="PROFILE")
+        
+        # Create managed browser instance
+        managed_browser = ManagedBrowser(
+            browser_type=browser_config.browser_type,
+            user_data_dir=profile_path,
+            headless=False,  # Must be visible
+            logger=self.logger,
+            debugging_port=browser_config.debugging_port
+        )
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="PROFILE")
+            await managed_browser.cleanup()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Profile creation interrupted. Profile may be incomplete.", tag="PROFILE")
+                sys.exit(1)
+                
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user is done with the browser
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' when you've finished using the browser...{Style.RESET_ALL}", tag="PROFILE")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser and saving profile...{Style.RESET_ALL}", tag="PROFILE")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if managed_browser.browser_process and managed_browser.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="PROFILE")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+        
+        try:
+            # Start the browser
+            await managed_browser.start()
+            
+            # Check if browser started successfully
+            browser_process = managed_browser.browser_process
+            if not browser_process:
+                self.logger.error("Failed to start browser process.", tag="PROFILE")
+                return None
+            
+            self.logger.info(f"Browser launched. {Fore.CYAN}Waiting for you to finish...{Style.RESET_ALL}", tag="PROFILE") 
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for either the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="PROFILE")
+                await managed_browser.cleanup()
+            
+            self.logger.success(f"Browser closed. Profile saved at: {Fore.GREEN}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+                
+        except Exception as e:
+            self.logger.error(f"Error creating profile: {str(e)}", tag="PROFILE")
+            await managed_browser.cleanup()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await managed_browser.cleanup()
+        
+        # Return the profile path
+        return profile_path
+    
+    def list_profiles(self) -> List[Dict[str, Any]]:
+        """
+        Lists all available browser profiles in the Crawl4AI profiles directory.
+        
+        Returns:
+            list: A list of dictionaries containing profile information:
+                  [{"name": "profile_name", "path": "/path/to/profile", "created": datetime, "type": "chromium|firefox"}]
+                  
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # List all available profiles
+            profiles = profiler.list_profiles()
+            
+            for profile in profiles:
+                print(f"Profile: {profile['name']}")
+                print(f"  Path: {profile['path']}")
+                print(f"  Created: {profile['created']}")
+                print(f"  Browser type: {profile['type']}")
+            ```
+        """
+        if not os.path.exists(self.profiles_dir):
+            return []
+            
+        profiles = []
+        
+        for name in os.listdir(self.profiles_dir):
+            profile_path = os.path.join(self.profiles_dir, name)
+            
+            # Skip if not a directory
+            if not os.path.isdir(profile_path):
+                continue
+                
+            # Check if this looks like a valid browser profile
+            # For Chromium: Look for Preferences file
+            # For Firefox: Look for prefs.js file
+            is_valid = False
+            
+            if os.path.exists(os.path.join(profile_path, "Preferences")) or \
+               os.path.exists(os.path.join(profile_path, "Default", "Preferences")):
+                is_valid = "chromium"
+            elif os.path.exists(os.path.join(profile_path, "prefs.js")):
+                is_valid = "firefox"
+                
+            if is_valid:
+                # Get creation time
+                created = datetime.datetime.fromtimestamp(
+                    os.path.getctime(profile_path)
+                )
+                
+                profiles.append({
+                    "name": name,
+                    "path": profile_path,
+                    "created": created,
+                    "type": is_valid
+                })
+                
+        # Sort by creation time, newest first
+        profiles.sort(key=lambda x: x["created"], reverse=True)
+        
+        return profiles
+    
+    def get_profile_path(self, profile_name: str) -> Optional[str]:
+        """
+        Get the full path to a profile by name.
+        
+        Args:
+            profile_name (str): Name of the profile (not the full path)
+            
+        Returns:
+            str: Full path to the profile directory, or None if not found
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            path = profiler.get_profile_path("my-profile")
+            if path:
+                print(f"Profile path: {path}")
+            else:
+                print("Profile not found")
+            ```
+        """
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            # Chrck if profile_name itself is full path
+            if os.path.isabs(profile_name):
+                profile_path = profile_name
+            else:
+                return None
+        
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return None  # Not a valid browser profile
+            
+        return profile_path
+    
+    def delete_profile(self, profile_name_or_path: str) -> bool:
+        """
+        Delete a browser profile by name or path.
+        
+        Args:
+            profile_name_or_path (str): Name of the profile or full path to profile directory
+            
+        Returns:
+            bool: True if the profile was deleted successfully, False otherwise
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Delete by name
+            success = profiler.delete_profile("my-profile")
+            
+            # Delete by path
+            success = profiler.delete_profile("/path/to/.crawl4ai/profiles/my-profile")
+            ```
+        """
+        # Determine if input is a name or a path
+        if os.path.isabs(profile_name_or_path):
+            # Full path provided
+            profile_path = profile_name_or_path
+        else:
+            # Just a name provided, construct path
+            profile_path = os.path.join(self.profiles_dir, profile_name_or_path)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            return False
+            
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return False  # Not a valid browser profile
+            
+        # Delete the profile directory
+        try:
+            shutil.rmtree(profile_path)
+            return True
+        except Exception:
+            return False
+            
+    async def interactive_manager(self, crawl_callback=None):
+        """
+        Launch an interactive profile management console.
+        
+        Args:
+            crawl_callback (callable, optional): Function to call when selecting option to use 
+                a profile for crawling. It will be called with (profile_path, url).
+                
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Define a custom crawl function
+            async def my_crawl_function(profile_path, url):
+                print(f"Crawling {url} with profile {profile_path}")
+                # Implement your crawling logic here
+                
+            # Start interactive manager
+            await profiler.interactive_manager(crawl_callback=my_crawl_function)
+            ```
+        """
+        while True:
+            self.logger.info(f"\n{Fore.CYAN}Profile Management Options:{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"1. {Fore.GREEN}Create a new profile{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"2. {Fore.YELLOW}List available profiles{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"3. {Fore.RED}Delete a profile{Style.RESET_ALL}", tag="MENU")
+            
+            # Only show crawl option if callback provided
+            if crawl_callback:
+                self.logger.info(f"4. {Fore.CYAN}Use a profile to crawl a website{Style.RESET_ALL}", tag="MENU")
+                self.logger.info(f"5. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "5"
+            else:
+                self.logger.info(f"4. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "4"
+            
+            choice = input(f"\n{Fore.CYAN}Enter your choice (1-{exit_option}): {Style.RESET_ALL}")
+            
+            if choice == "1":
+                # Create new profile
+                name = input(f"{Fore.GREEN}Enter a name for the new profile (or press Enter for auto-generated name): {Style.RESET_ALL}")
+                await self.create_profile(name or None)
+                
+            elif choice == "2":
+                # List profiles
+                profiles = self.list_profiles()
+                
+                if not profiles:
+                    self.logger.warning("  No profiles found. Create one first with option 1.", tag="PROFILES")
+                    continue
+                
+                # Print profile information with colorama formatting
+                self.logger.info("\nAvailable profiles:", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {Fore.CYAN}{profile['name']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Path: {Fore.YELLOW}{profile['path']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES")
+                    self.logger.info(f"    Browser type: {profile['type']}", tag="PROFILES")
+                    self.logger.info("", tag="PROFILES")  # Empty line for spacing
+                
+            elif choice == "3":
+                # Delete profile
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found to delete", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to delete
+                profile_idx = input(f"{Fore.RED}Enter the number of the profile to delete (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_name = profiles[idx]["name"]
+                        self.logger.info(f"Deleting profile: {Fore.YELLOW}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                        
+                        # Confirm deletion
+                        confirm = input(f"{Fore.RED}Are you sure you want to delete this profile? (y/n): {Style.RESET_ALL}")
+                        if confirm.lower() == 'y':
+                            success = self.delete_profile(profiles[idx]["path"])
+                            
+                            if success:
+                                self.logger.success(f"Profile {Fore.GREEN}{profile_name}{Style.RESET_ALL} deleted successfully", tag="PROFILES")
+                            else:
+                                self.logger.error(f"Failed to delete profile {Fore.RED}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == "4" and crawl_callback:
+                # Use profile to crawl a site
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found. Create one first.", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to use
+                profile_idx = input(f"{Fore.CYAN}Enter the number of the profile to use (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_path = profiles[idx]["path"]
+                        url = input(f"{Fore.CYAN}Enter the URL to crawl: {Style.RESET_ALL}")
+                        if url:
+                            # Call the provided crawl callback
+                            await crawl_callback(profile_path, url)
+                        else:
+                            self.logger.error("No URL provided", tag="CRAWL")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == exit_option:
+                # Exit
+                self.logger.info("Exiting profile management", tag="MENU")
+                break
+                
+            else:
+                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")
+
+    async def launch_standalone_browser(self, 
+                                  browser_type: str = "chromium",
+                                  user_data_dir: Optional[str] = None,
+                                  debugging_port: int = 9222,
+                                  headless: bool = False,
+                                  save_as_builtin: bool = False) -> Optional[str]:
+        """
+        Launch a standalone browser with CDP debugging enabled and keep it running
+        until the user presses 'q'. Returns and displays the CDP URL.
+        
+        Args:
+            browser_type (str): Type of browser to launch ('chromium' or 'firefox')
+            user_data_dir (str, optional): Path to user profile directory
+            debugging_port (int): Port to use for CDP debugging
+            headless (bool): Whether to run in headless mode
+            
+        Returns:
+            str: CDP URL for the browser, or None if launch failed
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            cdp_url = await profiler.launch_standalone_browser(
+                user_data_dir="/path/to/profile",
+                debugging_port=9222
+            )
+            # Use cdp_url to connect to the browser
+            ```
+        """
+        # Use the provided directory if specified, otherwise create a temporary directory
+        if user_data_dir:
+            # Directory is provided directly, ensure it exists
+            profile_path = user_data_dir
+            os.makedirs(profile_path, exist_ok=True)
+        else:
+            # Create a temporary profile directory
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"temp_{timestamp}_{uuid.uuid4().hex[:6]}"
+            profile_path = os.path.join(self.profiles_dir, profile_name)
+            os.makedirs(profile_path, exist_ok=True)
+        
+        # Print initial information
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="CDP")
+        self.logger.info(f"Launching standalone browser with CDP debugging", tag="CDP")
+        self.logger.info(f"Browser type: {Fore.GREEN}{browser_type}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Profile path: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Debugging port: {Fore.CYAN}{debugging_port}{Style.RESET_ALL}", tag="CDP")
+        self.logger.info(f"Headless mode: {Fore.CYAN}{headless}{Style.RESET_ALL}", tag="CDP")
+        
+        # Create managed browser instance
+        managed_browser = ManagedBrowser(
+            browser_type=browser_type,
+            user_data_dir=profile_path,
+            headless=headless,
+            logger=self.logger,
+            debugging_port=debugging_port
+        )
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="CDP")
+            await managed_browser.cleanup()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Browser terminated by user.", tag="CDP")
+                sys.exit(1)
+                    
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user wants to exit
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' to stop the browser and exit...{Style.RESET_ALL}", tag="CDP")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser...{Style.RESET_ALL}", tag="CDP")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if managed_browser.browser_process and managed_browser.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="CDP")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+                
+        # Function to retrieve and display CDP JSON config
+        async def get_cdp_json(port):
+            import aiohttp
+            cdp_url = f"http://localhost:{port}"
+            json_url = f"{cdp_url}/json/version"
+            
+            try:
+                async with aiohttp.ClientSession() as session:
+                    # Try multiple times in case the browser is still starting up
+                    for _ in range(10):
+                        try:
+                            async with session.get(json_url) as response:
+                                if response.status == 200:
+                                    data = await response.json()
+                                    return cdp_url, data
+                        except Exception:
+                            pass
+                        
+                        await asyncio.sleep(0.5)
+                    
+                    return cdp_url, None
+            except Exception as e:
+                self.logger.error(f"Error fetching CDP JSON: {str(e)}", tag="CDP")
+                return cdp_url, None
+        
+        cdp_url = None
+        config_json = None
+        
+        try:
+            # Start the browser
+            await managed_browser.start()
+            
+            # Check if browser started successfully
+            browser_process = managed_browser.browser_process
+            if not browser_process:
+                self.logger.error("Failed to start browser process.", tag="CDP")
+                return None
+            
+            self.logger.info(f"Browser launched successfully. Retrieving CDP information...", tag="CDP") 
+            
+            # Get CDP URL and JSON config
+            cdp_url, config_json = await get_cdp_json(debugging_port)
+            
+            if cdp_url:
+                self.logger.success(f"CDP URL: {Fore.GREEN}{cdp_url}{Style.RESET_ALL}", tag="CDP")
+                
+                if config_json:
+                    # Display relevant CDP information
+                    self.logger.info(f"Browser: {Fore.CYAN}{config_json.get('Browser', 'Unknown')}{Style.RESET_ALL}", tag="CDP")
+                    self.logger.info(f"Protocol Version: {config_json.get('Protocol-Version', 'Unknown')}", tag="CDP")
+                    if 'webSocketDebuggerUrl' in config_json:
+                        self.logger.info(f"WebSocket URL: {Fore.GREEN}{config_json['webSocketDebuggerUrl']}{Style.RESET_ALL}", tag="CDP")
+                else:
+                    self.logger.warning("Could not retrieve CDP configuration JSON", tag="CDP")
+            else:
+                self.logger.error(f"Failed to get CDP URL on port {debugging_port}", tag="CDP")
+                await managed_browser.cleanup()
+                return None
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="CDP")
+                await managed_browser.cleanup()
+            
+            self.logger.success(f"Browser closed.", tag="CDP")
+                
+        except Exception as e:
+            self.logger.error(f"Error launching standalone browser: {str(e)}", tag="CDP")
+            await managed_browser.cleanup()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await managed_browser.cleanup()
+        
+        # Return the CDP URL
+        return cdp_url
+    
+    async def launch_builtin_browser(self, 
+                                 browser_type: str = "chromium",
+                                 debugging_port: int = 9222,
+                                 headless: bool = True) -> Optional[str]:
+        """
+        Launch a browser in the background for use as the builtin browser.
+        
+        Args:
+            browser_type (str): Type of browser to launch ('chromium' or 'firefox')
+            debugging_port (int): Port to use for CDP debugging
+            headless (bool): Whether to run in headless mode
+            
+        Returns:
+            str: CDP URL for the browser, or None if launch failed
+        """
+        # Check if there's an existing browser still running
+        browser_info = self.get_builtin_browser_info()
+        if browser_info and self._is_browser_running(browser_info.get('pid')):
+            self.logger.info("Builtin browser is already running", tag="BUILTIN")
+            return browser_info.get('cdp_url')
+        
+        # Create a user data directory for the builtin browser
+        user_data_dir = os.path.join(self.builtin_browser_dir, "user_data")
+        os.makedirs(user_data_dir, exist_ok=True)
+        
+        # Create managed browser instance
+        managed_browser = ManagedBrowser(
+            browser_type=browser_type,
+            user_data_dir=user_data_dir,
+            headless=headless,
+            logger=self.logger,
+            debugging_port=debugging_port
+        )
+        
+        try:
+            # Start the browser
+            await managed_browser.start()
+            
+            # Check if browser started successfully
+            browser_process = managed_browser.browser_process
+            if not browser_process:
+                self.logger.error("Failed to start browser process.", tag="BUILTIN")
+                return None
+            
+            # Get CDP URL
+            cdp_url = f"http://localhost:{debugging_port}"
+            
+            # Try to verify browser is responsive by fetching version info
+            import aiohttp
+            json_url = f"{cdp_url}/json/version"
+            config_json = None
+            
+            try:
+                async with aiohttp.ClientSession() as session:
+                    for _ in range(10):  # Try multiple times
+                        try:
+                            async with session.get(json_url) as response:
+                                if response.status == 200:
+                                    config_json = await response.json()
+                                    break
+                        except Exception:
+                            pass
+                        await asyncio.sleep(0.5)
+            except Exception as e:
+                self.logger.warning(f"Could not verify browser: {str(e)}", tag="BUILTIN")
+            
+            # Save browser info
+            browser_info = {
+                'pid': browser_process.pid,
+                'cdp_url': cdp_url,
+                'user_data_dir': user_data_dir,
+                'browser_type': browser_type,
+                'debugging_port': debugging_port,
+                'start_time': time.time(),
+                'config': config_json
+            }
+            
+            with open(self.builtin_config_file, 'w') as f:
+                json.dump(browser_info, f, indent=2)
+                
+            # Detach from the browser process - don't keep any references
+            # This is important to allow the Python script to exit while the browser continues running
+            # We'll just record the PID and other info, and the browser will run independently
+            managed_browser.browser_process = None
+                
+            self.logger.success(f"Builtin browser launched at CDP URL: {cdp_url}", tag="BUILTIN")
+            return cdp_url
+            
+        except Exception as e:
+            self.logger.error(f"Error launching builtin browser: {str(e)}", tag="BUILTIN")
+            if managed_browser:
+                await managed_browser.cleanup()
+            return None
+    
+    def get_builtin_browser_info(self) -> Optional[Dict[str, Any]]:
+        """
+        Get information about the builtin browser.
+        
+        Returns:
+            dict: Browser information or None if no builtin browser is configured
+        """
+        if not os.path.exists(self.builtin_config_file):
+            return None
+            
+        try:
+            with open(self.builtin_config_file, 'r') as f:
+                browser_info = json.load(f)
+                
+            # Check if the browser is still running
+            if not self._is_browser_running(browser_info.get('pid')):
+                self.logger.warning("Builtin browser is not running", tag="BUILTIN")
+                return None
+                
+            return browser_info
+        except Exception as e:
+            self.logger.error(f"Error reading builtin browser config: {str(e)}", tag="BUILTIN")
+            return None
+            
+    def _is_browser_running(self, pid: Optional[int]) -> bool:
+        """Check if a process with the given PID is running"""
+        if not pid:
+            return False
+            
+        try:
+            # Check if the process exists
+            if sys.platform == "win32":
+                process = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], 
+                                         capture_output=True, text=True)
+                return str(pid) in process.stdout
+            else:
+                # Unix-like systems
+                os.kill(pid, 0)  # This doesn't actually kill the process, just checks if it exists
+            return True
+        except (ProcessLookupError, PermissionError, OSError):
+            return False
+            
+    async def kill_builtin_browser(self) -> bool:
+        """
+        Kill the builtin browser if it's running.
+        
+        Returns:
+            bool: True if the browser was killed, False otherwise
+        """
+        browser_info = self.get_builtin_browser_info()
+        if not browser_info:
+            self.logger.warning("No builtin browser found", tag="BUILTIN")
+            return False
+            
+        pid = browser_info.get('pid')
+        if not pid:
+            return False
+            
+        try:
+            if sys.platform == "win32":
+                subprocess.run(["taskkill", "/F", "/PID", str(pid)], check=True)
+            else:
+                os.kill(pid, signal.SIGTERM)
+                # Wait for termination
+                for _ in range(5):
+                    if not self._is_browser_running(pid):
+                        break
+                    await asyncio.sleep(0.5)
+                else:
+                    # Force kill if still running
+                    os.kill(pid, signal.SIGKILL)
+                    
+            # Remove config file
+            if os.path.exists(self.builtin_config_file):
+                os.unlink(self.builtin_config_file)
+                
+            self.logger.success("Builtin browser terminated", tag="BUILTIN")
+            return True
+        except Exception as e:
+            self.logger.error(f"Error killing builtin browser: {str(e)}", tag="BUILTIN")
+            return False
+    
+    async def get_builtin_browser_status(self) -> Dict[str, Any]:
+        """
+        Get status information about the builtin browser.
+        
+        Returns:
+            dict: Status information with running, cdp_url, and info fields
+        """
+        browser_info = self.get_builtin_browser_info()
+        
+        if not browser_info:
+            return {
+                'running': False,
+                'cdp_url': None,
+                'info': None
+            }
+            
+        return {
+            'running': True,
+            'cdp_url': browser_info.get('cdp_url'),
+            'info': browser_info
+        }
+

crawl4ai/chunking_strategy.py

@@ -4,7 +4,6 @@ from collections import Counter
 import string
 from .model_loader import load_nltk_punkt
 
-
 # Define the abstract base class for chunking strategies
 class ChunkingStrategy(ABC):
     """
@@ -72,6 +71,7 @@ class NlpSentenceChunking(ChunkingStrategy):
         """
         Initialize the NlpSentenceChunking object.
         """
+        from crawl4ai.le.legacy.model_loader import load_nltk_punkt
         load_nltk_punkt()
 
     def chunk(self, text: str) -> list:

crawl4ai/components/crawler_monitor.py

@@ -0,0 +1,837 @@
+import time
+import uuid
+import threading
+import psutil
+from datetime import datetime, timedelta
+from typing import Dict, Optional, List
+import threading
+from rich.console import Console
+from rich.layout import Layout
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+from rich.live import Live
+from rich import box
+from ..models import CrawlStatus
+
+class TerminalUI:
+    """Terminal user interface for CrawlerMonitor using rich library."""
+    
+    def __init__(self, refresh_rate: float = 1.0, max_width: int = 120):
+        """
+        Initialize the terminal UI.
+        
+        Args:
+            refresh_rate: How often to refresh the UI (in seconds)
+            max_width: Maximum width of the UI in characters
+        """
+        self.console = Console(width=max_width)
+        self.layout = Layout()
+        self.refresh_rate = refresh_rate
+        self.stop_event = threading.Event()
+        self.ui_thread = None
+        self.monitor = None  # Will be set by CrawlerMonitor
+        self.max_width = max_width
+        
+        # Setup layout - vertical layout (top to bottom)
+        self.layout.split(
+            Layout(name="header", size=3),
+            Layout(name="pipeline_status", size=10),
+            Layout(name="task_details", ratio=1),
+            Layout(name="footer", size=3)  # Increased footer size to fit all content
+        )
+        
+    def start(self, monitor):
+        """Start the UI thread."""
+        self.monitor = monitor
+        self.stop_event.clear()
+        self.ui_thread = threading.Thread(target=self._ui_loop)
+        self.ui_thread.daemon = True
+        self.ui_thread.start()
+        
+    def stop(self):
+        """Stop the UI thread."""
+        if self.ui_thread and self.ui_thread.is_alive():
+            self.stop_event.set()
+            # Only try to join if we're not in the UI thread
+            # This prevents "cannot join current thread" errors
+            if threading.current_thread() != self.ui_thread:
+                self.ui_thread.join(timeout=5.0)
+    
+    def _ui_loop(self):
+        """Main UI rendering loop."""
+        import sys
+        import select
+        import termios
+        import tty
+        
+        # Setup terminal for non-blocking input
+        old_settings = termios.tcgetattr(sys.stdin)
+        try:
+            tty.setcbreak(sys.stdin.fileno())
+            
+            # Use Live display to render the UI
+            with Live(self.layout, refresh_per_second=1/self.refresh_rate, screen=True) as live:
+                self.live = live  # Store the live display for updates
+                
+                # Main UI loop
+                while not self.stop_event.is_set():
+                    self._update_display()
+                    
+                    # Check for key press (non-blocking)
+                    if select.select([sys.stdin], [], [], 0)[0]:
+                        key = sys.stdin.read(1)
+                        # Check for 'q' to quit
+                        if key == 'q':
+                            # Signal stop but don't call monitor.stop() from UI thread
+                            # as it would cause the thread to try to join itself
+                            self.stop_event.set()
+                            self.monitor.is_running = False
+                            break
+                    
+                    time.sleep(self.refresh_rate)
+                    
+                    # Just check if the monitor was stopped
+                    if not self.monitor.is_running:
+                        break
+        finally:
+            # Restore terminal settings
+            termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
+    
+    def _update_display(self):
+        """Update the terminal display with current statistics."""
+        if not self.monitor:
+            return
+            
+        # Update crawler status panel
+        self.layout["header"].update(self._create_status_panel())
+        
+        # Update pipeline status panel and task details panel
+        self.layout["pipeline_status"].update(self._create_pipeline_panel())
+        self.layout["task_details"].update(self._create_task_details_panel())
+        
+        # Update footer
+        self.layout["footer"].update(self._create_footer())
+    
+    def _create_status_panel(self) -> Panel:
+        """Create the crawler status panel."""
+        summary = self.monitor.get_summary()
+        
+        # Format memory status with icon
+        memory_status = self.monitor.get_memory_status()
+        memory_icon = "🟢"  # Default NORMAL
+        if memory_status == "PRESSURE":
+            memory_icon = "🟠"
+        elif memory_status == "CRITICAL":
+            memory_icon = "🔴"
+        
+        # Get current memory usage
+        current_memory = psutil.Process().memory_info().rss / (1024 * 1024)  # MB
+        memory_percent = (current_memory / psutil.virtual_memory().total) * 100
+        
+        # Format runtime
+        runtime = self.monitor._format_time(time.time() - self.monitor.start_time if self.monitor.start_time else 0)
+        
+        # Create the status text
+        status_text = Text()
+        status_text.append(f"Web Crawler Dashboard | Runtime: {runtime} | Memory: {memory_percent:.1f}% {memory_icon}\n")
+        status_text.append(f"Status: {memory_status} | URLs: {summary['urls_completed']}/{summary['urls_total']} | ")
+        status_text.append(f"Peak Mem: {summary['peak_memory_percent']:.1f}% at {self.monitor._format_time(summary['peak_memory_time'])}")
+        
+        return Panel(status_text, title="Crawler Status", border_style="blue")
+    
+    def _create_pipeline_panel(self) -> Panel:
+        """Create the pipeline status panel."""
+        summary = self.monitor.get_summary()
+        queue_stats = self.monitor.get_queue_stats()
+        
+        # Create a table for status counts
+        table = Table(show_header=True, box=None)
+        table.add_column("Status", style="cyan")
+        table.add_column("Count", justify="right")
+        table.add_column("Percentage", justify="right")
+        table.add_column("Stat", style="cyan")
+        table.add_column("Value", justify="right")
+        
+        # Calculate overall progress
+        progress = f"{summary['urls_completed']}/{summary['urls_total']}"
+        progress_percent = f"{summary['completion_percentage']:.1f}%"
+        
+        # Add rows for each status
+        table.add_row(
+            "Overall Progress", 
+            progress, 
+            progress_percent,
+            "Est. Completion", 
+            summary.get('estimated_completion_time', "N/A")
+        )
+        
+        # Add rows for each status
+        status_counts = summary['status_counts']
+        total = summary['urls_total'] or 1  # Avoid division by zero
+        
+        # Status rows
+        table.add_row(
+            "Completed", 
+            str(status_counts.get(CrawlStatus.COMPLETED.name, 0)),
+            f"{status_counts.get(CrawlStatus.COMPLETED.name, 0) / total * 100:.1f}%",
+            "Avg. Time/URL",
+            f"{summary.get('avg_task_duration', 0):.2f}s"
+        )
+        
+        table.add_row(
+            "Failed", 
+            str(status_counts.get(CrawlStatus.FAILED.name, 0)),
+            f"{status_counts.get(CrawlStatus.FAILED.name, 0) / total * 100:.1f}%",
+            "Concurrent Tasks",
+            str(status_counts.get(CrawlStatus.IN_PROGRESS.name, 0))
+        )
+        
+        table.add_row(
+            "In Progress", 
+            str(status_counts.get(CrawlStatus.IN_PROGRESS.name, 0)),
+            f"{status_counts.get(CrawlStatus.IN_PROGRESS.name, 0) / total * 100:.1f}%",
+            "Queue Size",
+            str(queue_stats['total_queued'])
+        )
+        
+        table.add_row(
+            "Queued", 
+            str(status_counts.get(CrawlStatus.QUEUED.name, 0)),
+            f"{status_counts.get(CrawlStatus.QUEUED.name, 0) / total * 100:.1f}%",
+            "Max Wait Time",
+            f"{queue_stats['highest_wait_time']:.1f}s"
+        )
+        
+        # Requeued is a special case as it's not a status
+        requeued_count = summary.get('requeued_count', 0)
+        table.add_row(
+            "Requeued", 
+            str(requeued_count),
+            f"{summary.get('requeue_rate', 0):.1f}%",
+            "Avg Wait Time",
+            f"{queue_stats['avg_wait_time']:.1f}s"
+        )
+        
+        # Add empty row for spacing
+        table.add_row(
+            "", 
+            "",
+            "",
+            "Requeue Rate",
+            f"{summary.get('requeue_rate', 0):.1f}%"
+        )
+        
+        return Panel(table, title="Pipeline Status", border_style="green")
+    
+    def _create_task_details_panel(self) -> Panel:
+        """Create the task details panel."""
+        # Create a table for task details
+        table = Table(show_header=True, expand=True)
+        table.add_column("Task ID", style="cyan", no_wrap=True, width=10)
+        table.add_column("URL", style="blue", ratio=3)
+        table.add_column("Status", style="green", width=15)
+        table.add_column("Memory", justify="right", width=8)
+        table.add_column("Peak", justify="right", width=8)
+        table.add_column("Duration", justify="right", width=10)
+        
+        # Get all task stats
+        task_stats = self.monitor.get_all_task_stats()
+        
+        # Add summary row
+        active_tasks = sum(1 for stats in task_stats.values() 
+                          if stats['status'] == CrawlStatus.IN_PROGRESS.name)
+        
+        total_memory = sum(stats['memory_usage'] for stats in task_stats.values())
+        total_peak = sum(stats['peak_memory'] for stats in task_stats.values())
+        
+        # Summary row with separators
+        table.add_row(
+            "SUMMARY", 
+            f"Total: {len(task_stats)}", 
+            f"Active: {active_tasks}",
+            f"{total_memory:.1f}",
+            f"{total_peak:.1f}",
+            "N/A"
+        )
+        
+        # Add a separator
+        table.add_row("—" * 10, "—" * 20, "—" * 10, "—" * 8, "—" * 8, "—" * 10)
+        
+        # Status icons
+        status_icons = {
+            CrawlStatus.QUEUED.name: "⏳",
+            CrawlStatus.IN_PROGRESS.name: "🔄",
+            CrawlStatus.COMPLETED.name: "✅",
+            CrawlStatus.FAILED.name: "❌"
+        }
+        
+        # Calculate how many rows we can display based on available space
+        # We can display more rows now that we have a dedicated panel
+        display_count = min(len(task_stats), 20)  # Display up to 20 tasks
+        
+        # Add rows for each task
+        for task_id, stats in sorted(
+            list(task_stats.items())[:display_count],
+            # Sort: 1. IN_PROGRESS first, 2. QUEUED, 3. COMPLETED/FAILED by recency
+            key=lambda x: (
+                0 if x[1]['status'] == CrawlStatus.IN_PROGRESS.name else 
+                1 if x[1]['status'] == CrawlStatus.QUEUED.name else 
+                2,
+                -1 * (x[1].get('end_time', 0) or 0)  # Most recent first
+            )
+        ):
+            # Truncate task_id and URL for display
+            short_id = task_id[:8]
+            url = stats['url']
+            if len(url) > 50:  # Allow longer URLs in the dedicated panel
+                url = url[:47] + "..."
+                
+            # Format status with icon
+            status = f"{status_icons.get(stats['status'], '?')} {stats['status']}"
+            
+            # Add row
+            table.add_row(
+                short_id,
+                url,
+                status,
+                f"{stats['memory_usage']:.1f}",
+                f"{stats['peak_memory']:.1f}",
+                stats['duration'] if 'duration' in stats else "0:00"
+            )
+        
+        return Panel(table, title="Task Details", border_style="yellow")
+    
+    def _create_footer(self) -> Panel:
+        """Create the footer panel."""
+        from rich.columns import Columns
+        from rich.align import Align
+        
+        memory_status = self.monitor.get_memory_status()
+        memory_icon = "🟢"  # Default NORMAL
+        if memory_status == "PRESSURE":
+            memory_icon = "🟠"
+        elif memory_status == "CRITICAL":
+            memory_icon = "🔴"
+        
+        # Left section - memory status
+        left_text = Text()
+        left_text.append("Memory Status: ", style="bold")
+        status_style = "green" if memory_status == "NORMAL" else "yellow" if memory_status == "PRESSURE" else "red bold"
+        left_text.append(f"{memory_icon} {memory_status}", style=status_style)
+        
+        # Center section - copyright
+        center_text = Text("© Crawl4AI 2025 | Made by UnclecCode", style="cyan italic")
+        
+        # Right section - quit instruction
+        right_text = Text()
+        right_text.append("Press ", style="bold")
+        right_text.append("q", style="white on blue")
+        right_text.append(" to quit", style="bold")
+        
+        # Create columns with the three sections
+        footer_content = Columns(
+            [
+                Align.left(left_text),
+                Align.center(center_text),
+                Align.right(right_text)
+            ],
+            expand=True
+        )
+        
+        # Create a more visible footer panel
+        return Panel(
+            footer_content,
+            border_style="white",
+            padding=(0, 1)  # Add padding for better visibility
+        )
+
+
+class CrawlerMonitor:
+    """
+    Comprehensive monitoring and visualization system for tracking web crawler operations in real-time.
+    Provides a terminal-based dashboard that displays task statuses, memory usage, queue statistics,
+    and performance metrics.
+    """
+    
+    def __init__(
+        self,
+        urls_total: int = 0,
+        refresh_rate: float = 1.0,
+        enable_ui: bool = True,
+        max_width: int = 120
+    ):
+        """
+        Initialize the CrawlerMonitor.
+        
+        Args:
+            urls_total: Total number of URLs to be crawled
+            refresh_rate: How often to refresh the UI (in seconds)
+            enable_ui: Whether to display the terminal UI
+            max_width: Maximum width of the UI in characters
+        """
+        # Core monitoring attributes
+        self.stats = {}  # Task ID -> stats dict
+        self.memory_status = "NORMAL"
+        self.start_time = None
+        self.end_time = None
+        self.is_running = False
+        self.queue_stats = {
+            "total_queued": 0,
+            "highest_wait_time": 0.0,
+            "avg_wait_time": 0.0
+        }
+        self.urls_total = urls_total
+        self.urls_completed = 0
+        self.peak_memory_percent = 0.0
+        self.peak_memory_time = 0.0
+        
+        # Status counts
+        self.status_counts = {
+            CrawlStatus.QUEUED.name: 0,
+            CrawlStatus.IN_PROGRESS.name: 0,
+            CrawlStatus.COMPLETED.name: 0,
+            CrawlStatus.FAILED.name: 0
+        }
+        
+        # Requeue tracking
+        self.requeued_count = 0
+        
+        # Thread-safety
+        self._lock = threading.RLock()
+        
+        # Terminal UI
+        self.enable_ui = enable_ui
+        self.terminal_ui = TerminalUI(
+            refresh_rate=refresh_rate, 
+            max_width=max_width
+        ) if enable_ui else None
+    
+    def start(self):
+        """
+        Start the monitoring session.
+        
+        - Initializes the start_time
+        - Sets is_running to True
+        - Starts the terminal UI if enabled
+        """
+        with self._lock:
+            self.start_time = time.time()
+            self.is_running = True
+            
+            # Start the terminal UI
+            if self.enable_ui and self.terminal_ui:
+                self.terminal_ui.start(self)
+    
+    def stop(self):
+        """
+        Stop the monitoring session.
+        
+        - Records end_time
+        - Sets is_running to False
+        - Stops the terminal UI
+        - Generates final summary statistics
+        """
+        with self._lock:
+            self.end_time = time.time()
+            self.is_running = False
+            
+            # Stop the terminal UI
+            if self.enable_ui and self.terminal_ui:
+                self.terminal_ui.stop()
+    
+    def add_task(self, task_id: str, url: str):
+        """
+        Register a new task with the monitor.
+        
+        Args:
+            task_id: Unique identifier for the task
+            url: URL being crawled
+            
+        The task is initialized with:
+            - status: QUEUED
+            - url: The URL to crawl
+            - enqueue_time: Current time
+            - memory_usage: 0
+            - peak_memory: 0
+            - wait_time: 0
+            - retry_count: 0
+        """
+        with self._lock:
+            self.stats[task_id] = {
+                "task_id": task_id,
+                "url": url,
+                "status": CrawlStatus.QUEUED.name,
+                "enqueue_time": time.time(),
+                "start_time": None,
+                "end_time": None,
+                "memory_usage": 0.0,
+                "peak_memory": 0.0,
+                "error_message": "",
+                "wait_time": 0.0,
+                "retry_count": 0,
+                "duration": "0:00",
+                "counted_requeue": False
+            }
+            
+            # Update status counts
+            self.status_counts[CrawlStatus.QUEUED.name] += 1
+    
+    def update_task(
+        self, 
+        task_id: str, 
+        status: Optional[CrawlStatus] = None,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None,
+        memory_usage: Optional[float] = None,
+        peak_memory: Optional[float] = None,
+        error_message: Optional[str] = None,
+        retry_count: Optional[int] = None,
+        wait_time: Optional[float] = None
+    ):
+        """
+        Update statistics for a specific task.
+        
+        Args:
+            task_id: Unique identifier for the task
+            status: New status (QUEUED, IN_PROGRESS, COMPLETED, FAILED)
+            start_time: When task execution started
+            end_time: When task execution ended
+            memory_usage: Current memory usage in MB
+            peak_memory: Maximum memory usage in MB
+            error_message: Error description if failed
+            retry_count: Number of retry attempts
+            wait_time: Time spent in queue
+            
+        Updates task statistics and updates status counts.
+        If status changes, decrements old status count and 
+        increments new status count.
+        """
+        with self._lock:
+            # Check if task exists
+            if task_id not in self.stats:
+                return
+            
+            task_stats = self.stats[task_id]
+            
+            # Update status counts if status is changing
+            old_status = task_stats["status"]
+            if status and status.name != old_status:
+                self.status_counts[old_status] -= 1
+                self.status_counts[status.name] += 1
+                
+                # Track completion
+                if status == CrawlStatus.COMPLETED:
+                    self.urls_completed += 1
+                
+                # Track requeues
+                if old_status in [CrawlStatus.COMPLETED.name, CrawlStatus.FAILED.name] and not task_stats.get("counted_requeue", False):
+                    self.requeued_count += 1
+                    task_stats["counted_requeue"] = True
+            
+            # Update task statistics
+            if status:
+                task_stats["status"] = status.name
+            if start_time is not None:
+                task_stats["start_time"] = start_time
+            if end_time is not None:
+                task_stats["end_time"] = end_time
+            if memory_usage is not None:
+                task_stats["memory_usage"] = memory_usage
+                
+                # Update peak memory if necessary
+                current_percent = (memory_usage / psutil.virtual_memory().total) * 100
+                if current_percent > self.peak_memory_percent:
+                    self.peak_memory_percent = current_percent
+                    self.peak_memory_time = time.time()
+                
+            if peak_memory is not None:
+                task_stats["peak_memory"] = peak_memory
+            if error_message is not None:
+                task_stats["error_message"] = error_message
+            if retry_count is not None:
+                task_stats["retry_count"] = retry_count
+            if wait_time is not None:
+                task_stats["wait_time"] = wait_time
+            
+            # Calculate duration
+            if task_stats["start_time"]:
+                end = task_stats["end_time"] or time.time()
+                duration = end - task_stats["start_time"]
+                task_stats["duration"] = self._format_time(duration)
+    
+    def update_memory_status(self, status: str):
+        """
+        Update the current memory status.
+        
+        Args:
+            status: Memory status (NORMAL, PRESSURE, CRITICAL, or custom)
+            
+        Also updates the UI to reflect the new status.
+        """
+        with self._lock:
+            self.memory_status = status
+    
+    def update_queue_statistics(
+        self,
+        total_queued: int,
+        highest_wait_time: float,
+        avg_wait_time: float
+    ):
+        """
+        Update statistics related to the task queue.
+        
+        Args:
+            total_queued: Number of tasks currently in queue
+            highest_wait_time: Longest wait time of any queued task
+            avg_wait_time: Average wait time across all queued tasks
+        """
+        with self._lock:
+            self.queue_stats = {
+                "total_queued": total_queued,
+                "highest_wait_time": highest_wait_time,
+                "avg_wait_time": avg_wait_time
+            }
+    
+    def get_task_stats(self, task_id: str) -> Dict:
+        """
+        Get statistics for a specific task.
+        
+        Args:
+            task_id: Unique identifier for the task
+            
+        Returns:
+            Dictionary containing all task statistics
+        """
+        with self._lock:
+            return self.stats.get(task_id, {}).copy()
+    
+    def get_all_task_stats(self) -> Dict[str, Dict]:
+        """
+        Get statistics for all tasks.
+        
+        Returns:
+            Dictionary mapping task_ids to their statistics
+        """
+        with self._lock:
+            return self.stats.copy()
+    
+    def get_memory_status(self) -> str:
+        """
+        Get the current memory status.
+        
+        Returns:
+            Current memory status string
+        """
+        with self._lock:
+            return self.memory_status
+    
+    def get_queue_stats(self) -> Dict:
+        """
+        Get current queue statistics.
+        
+        Returns:
+            Dictionary with queue statistics including:
+            - total_queued: Number of tasks in queue
+            - highest_wait_time: Longest wait time
+            - avg_wait_time: Average wait time
+        """
+        with self._lock:
+            return self.queue_stats.copy()
+    
+    def get_summary(self) -> Dict:
+        """
+        Get a summary of all crawler statistics.
+        
+        Returns:
+            Dictionary containing:
+            - runtime: Total runtime in seconds
+            - urls_total: Total URLs to process
+            - urls_completed: Number of completed URLs
+            - completion_percentage: Percentage complete
+            - status_counts: Count of tasks in each status
+            - memory_status: Current memory status
+            - peak_memory_percent: Highest memory usage
+            - peak_memory_time: When peak memory occurred
+            - avg_task_duration: Average task processing time
+            - estimated_completion_time: Projected finish time
+            - requeue_rate: Percentage of tasks requeued
+        """
+        with self._lock:
+            # Calculate runtime
+            current_time = time.time()
+            runtime = current_time - (self.start_time or current_time)
+            
+            # Calculate completion percentage
+            completion_percentage = 0
+            if self.urls_total > 0:
+                completion_percentage = (self.urls_completed / self.urls_total) * 100
+            
+            # Calculate average task duration for completed tasks
+            completed_tasks = [
+                task for task in self.stats.values() 
+                if task["status"] == CrawlStatus.COMPLETED.name and task.get("start_time") and task.get("end_time")
+            ]
+            
+            avg_task_duration = 0
+            if completed_tasks:
+                total_duration = sum(task["end_time"] - task["start_time"] for task in completed_tasks)
+                avg_task_duration = total_duration / len(completed_tasks)
+            
+            # Calculate requeue rate
+            requeue_rate = 0
+            if len(self.stats) > 0:
+                requeue_rate = (self.requeued_count / len(self.stats)) * 100
+            
+            # Calculate estimated completion time
+            estimated_completion_time = "N/A"
+            if avg_task_duration > 0 and self.urls_total > 0 and self.urls_completed > 0:
+                remaining_tasks = self.urls_total - self.urls_completed
+                estimated_seconds = remaining_tasks * avg_task_duration
+                estimated_completion_time = self._format_time(estimated_seconds)
+            
+            return {
+                "runtime": runtime,
+                "urls_total": self.urls_total,
+                "urls_completed": self.urls_completed,
+                "completion_percentage": completion_percentage,
+                "status_counts": self.status_counts.copy(),
+                "memory_status": self.memory_status,
+                "peak_memory_percent": self.peak_memory_percent,
+                "peak_memory_time": self.peak_memory_time,
+                "avg_task_duration": avg_task_duration,
+                "estimated_completion_time": estimated_completion_time,
+                "requeue_rate": requeue_rate,
+                "requeued_count": self.requeued_count
+            }
+    
+    def render(self):
+        """
+        Render the terminal UI.
+        
+        This is the main UI rendering loop that:
+        1. Updates all statistics
+        2. Formats the display
+        3. Renders the ASCII interface
+        4. Handles keyboard input
+        
+        Note: The actual rendering is handled by the TerminalUI class
+        which uses the rich library's Live display.
+        """
+        if self.enable_ui and self.terminal_ui:
+            # Force an update of the UI
+            if hasattr(self.terminal_ui, '_update_display'):
+                self.terminal_ui._update_display()
+    
+    def _format_time(self, seconds: float) -> str:
+        """
+        Format time in hours:minutes:seconds.
+        
+        Args:
+            seconds: Time in seconds
+            
+        Returns:
+            Formatted time string (e.g., "1:23:45")
+        """
+        delta = timedelta(seconds=int(seconds))
+        hours, remainder = divmod(delta.seconds, 3600)
+        minutes, seconds = divmod(remainder, 60)
+        
+        if hours > 0:
+            return f"{hours}:{minutes:02}:{seconds:02}"
+        else:
+            return f"{minutes}:{seconds:02}"
+    
+    def _calculate_estimated_completion(self) -> str:
+        """
+        Calculate estimated completion time based on current progress.
+        
+        Returns:
+            Formatted time string
+        """
+        summary = self.get_summary()
+        return summary.get("estimated_completion_time", "N/A")
+
+
+# Example code for testing
+if __name__ == "__main__":
+    # Initialize the monitor
+    monitor = CrawlerMonitor(urls_total=100)
+    
+    # Start monitoring
+    monitor.start()
+    
+    try:
+        # Simulate some tasks
+        for i in range(20):
+            task_id = str(uuid.uuid4())
+            url = f"https://example.com/page{i}"
+            monitor.add_task(task_id, url)
+            
+            # Simulate 20% of tasks are already running
+            if i < 4:
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.IN_PROGRESS,
+                    start_time=time.time() - 30,  # Started 30 seconds ago
+                    memory_usage=10.5
+                )
+                
+            # Simulate 10% of tasks are completed
+            if i >= 4 and i < 6:
+                start_time = time.time() - 60
+                end_time = time.time() - 15
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.IN_PROGRESS,
+                    start_time=start_time,
+                    memory_usage=8.2
+                )
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.COMPLETED,
+                    end_time=end_time,
+                    memory_usage=0,
+                    peak_memory=15.7
+                )
+                
+            # Simulate 5% of tasks fail
+            if i >= 6 and i < 7:
+                start_time = time.time() - 45
+                end_time = time.time() - 20
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.IN_PROGRESS,
+                    start_time=start_time,
+                    memory_usage=12.3
+                )
+                monitor.update_task(
+                    task_id=task_id,
+                    status=CrawlStatus.FAILED,
+                    end_time=end_time,
+                    memory_usage=0,
+                    peak_memory=18.2,
+                    error_message="Connection timeout"
+                )
+        
+        # Simulate memory pressure
+        monitor.update_memory_status("PRESSURE")
+        
+        # Simulate queue statistics
+        monitor.update_queue_statistics(
+            total_queued=16,  # 20 - 4 (in progress)
+            highest_wait_time=120.5,
+            avg_wait_time=60.2
+        )
+        
+        # Keep the monitor running for a demonstration
+        print("Crawler Monitor is running. Press 'q' to exit.")
+        while monitor.is_running:
+            time.sleep(0.1)
+            
+    except KeyboardInterrupt:
+        print("\nExiting crawler monitor...")
+    finally:
+        # Stop the monitor
+        monitor.stop()
+        print("Crawler monitor exited successfully.")

crawl4ai/config.py

@@ -4,7 +4,8 @@ from dotenv import load_dotenv
 load_dotenv()  # Load environment variables from .env file
 
 # Default provider, ONLY used when the extraction strategy is LLMExtractionStrategy
-DEFAULT_PROVIDER = "openai/gpt-4o-mini"
+DEFAULT_PROVIDER = "openai/gpt-4o"
+DEFAULT_PROVIDER_API_KEY = "OPENAI_API_KEY"
 MODEL_REPO_BRANCH = "new-release-0.0.2"
 # Provider-model dictionary, ONLY used when the extraction strategy is LLMExtractionStrategy
 PROVIDER_MODELS = {
@@ -15,10 +16,18 @@ PROVIDER_MODELS = {
     "openai/gpt-4o": os.getenv("OPENAI_API_KEY"),
     "openai/o1-mini": os.getenv("OPENAI_API_KEY"),
     "openai/o1-preview": os.getenv("OPENAI_API_KEY"),
+    "openai/o3-mini": os.getenv("OPENAI_API_KEY"),
+    "openai/o3-mini-high": os.getenv("OPENAI_API_KEY"),
     "anthropic/claude-3-haiku-20240307": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-opus-20240229": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-sonnet-20240229": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-5-sonnet-20240620": os.getenv("ANTHROPIC_API_KEY"),
+    "gemini/gemini-pro": os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-1.5-pro': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash-exp': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash-lite-preview-02-05': os.getenv("GEMINI_API_KEY"),
+    "deepseek/deepseek-chat": os.getenv("DEEPSEEK_API_KEY"),
 }
 
 # Chunk token threshold
@@ -84,3 +93,46 @@ SHOW_DEPRECATION_WARNINGS = True
 SCREENSHOT_HEIGHT_TRESHOLD = 10000
 PAGE_TIMEOUT = 60000
 DOWNLOAD_PAGE_TIMEOUT = 60000
+
+# Global user settings with descriptions and default values
+USER_SETTINGS = {
+    "DEFAULT_LLM_PROVIDER": {
+        "default": "openai/gpt-4o",
+        "description": "Default LLM provider in 'company/model' format (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')",
+        "type": "string"
+    },
+    "DEFAULT_LLM_PROVIDER_TOKEN": {
+        "default": "",
+        "description": "API token for the default LLM provider",
+        "type": "string",
+        "secret": True
+    },
+    "VERBOSE": {
+        "default": False,
+        "description": "Enable verbose output for all commands",
+        "type": "boolean"
+    },
+    "BROWSER_HEADLESS": {
+        "default": True,
+        "description": "Run browser in headless mode by default",
+        "type": "boolean"
+    },
+    "BROWSER_TYPE": {
+        "default": "chromium",
+        "description": "Default browser type (chromium or firefox)",
+        "type": "string",
+        "options": ["chromium", "firefox"]
+    },
+    "CACHE_MODE": {
+        "default": "bypass",
+        "description": "Default cache mode (bypass, use, or refresh)",
+        "type": "string",
+        "options": ["bypass", "use", "refresh"]
+    },
+    "USER_AGENT_MODE": {
+        "default": "default",
+        "description": "Default user agent mode (default, random, or mobile)",
+        "type": "string",
+        "options": ["default", "random", "mobile"]
+    }
+}

crawl4ai/content_filter_strategy.py

@@ -1,3 +1,4 @@
+import inspect
 import re
 import time
 from bs4 import BeautifulSoup, Tag
@@ -5,25 +6,47 @@ from typing import List, Tuple, Dict, Optional
 from rank_bm25 import BM25Okapi
 from collections import deque
 from bs4 import NavigableString, Comment
-from .utils import clean_tokens, perform_completion_with_backoff, escape_json_string, sanitize_html, get_home_folder, extract_xml_data
+
+from .utils import (
+    clean_tokens,
+    perform_completion_with_backoff,
+    escape_json_string,
+    sanitize_html,
+    get_home_folder,
+    extract_xml_data,
+    merge_chunks,
+)
+from .types import LLMConfig
+from .config import DEFAULT_PROVIDER, OVERLAP_RATE, WORD_TOKEN_RATE
 from abc import ABC, abstractmethod
 import math
 from snowballstemmer import stemmer
-from .config import DEFAULT_PROVIDER, OVERLAP_RATE, WORD_TOKEN_RATE
 from .models import TokenUsage
 from .prompts import PROMPT_FILTER_CONTENT
-import os
 import json
 import hashlib
 from pathlib import Path
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor
 from .async_logger import AsyncLogger, LogLevel
-from colorama import Fore, Style, init
+from colorama import Fore, Style
+
 
 class RelevantContentFilter(ABC):
     """Abstract base class for content filtering strategies"""
 
-    def __init__(self, user_query: str = None):
+    def __init__(
+        self,
+        user_query: str = None,
+        verbose: bool = False,
+        logger: Optional[AsyncLogger] = None,
+    ):
+        """
+        Initializes the RelevantContentFilter class with optional user query.
+
+        Args:
+            user_query (str): User query for filtering (optional).
+            verbose (bool): Enable verbose logging (default: False).
+        """
         self.user_query = user_query
         self.included_tags = {
             # Primary structure
@@ -92,6 +115,8 @@ class RelevantContentFilter(ABC):
             r"nav|footer|header|sidebar|ads|comment|promo|advert|social|share", re.I
         )
         self.min_word_count = 2
+        self.verbose = False
+        self.logger = logger
 
     @abstractmethod
     def filter_content(self, html: str) -> List[str]:
@@ -353,6 +378,7 @@ class RelevantContentFilter(ABC):
         except Exception:
             return str(tag)  # Fallback to original if anything fails
 
+
 class BM25ContentFilter(RelevantContentFilter):
     """
     Content filtering using BM25 algorithm with priority tag handling.
@@ -495,6 +521,7 @@ class BM25ContentFilter(RelevantContentFilter):
 
         return [self.clean_element(tag) for _, _, tag in selected_candidates]
 
+
 class PruningContentFilter(RelevantContentFilter):
     """
     Content filtering using pruning algorithm with dynamic threshold.
@@ -741,110 +768,131 @@ class PruningContentFilter(RelevantContentFilter):
                 class_id_score -= 0.5
         return class_id_score
 
+
 class LLMContentFilter(RelevantContentFilter):
-    """Content filtering using LLMs to generate relevant markdown."""
+    """Content filtering using LLMs to generate relevant markdown.
+
+    How it works:
+    1. Extracts page metadata with fallbacks.
+    2. Extracts text chunks from the body element.
+    3. Applies LLMs to generate markdown for each chunk.
+    4. Filters out chunks below the threshold.
+    5. Sorts chunks by score in descending order.
+    6. Returns the top N chunks.
+
+    Attributes:
+        llm_config (LLMConfig): LLM configuration object.
+        instruction (str): Instruction for LLM markdown generation
+        chunk_token_threshold (int): Chunk token threshold for splitting (default: 1e9).
+        overlap_rate (float): Overlap rate for chunking (default: 0.5).
+        word_token_rate (float): Word token rate for chunking (default: 0.2).
+        verbose (bool): Enable verbose logging (default: False).
+        logger (AsyncLogger): Custom logger for LLM operations (optional).
+    """
+    _UNWANTED_PROPS = {
+        'provider' : 'Instead, use llm_config=LLMConfig(provider="...")',
+        'api_token' : 'Instead, use llm_config=LlMConfig(api_token="...")',
+        'base_url' : 'Instead, use llm_config=LLMConfig(base_url="...")',
+        'api_base' : 'Instead, use llm_config=LLMConfig(base_url="...")',
+    }
 
     def __init__(
         self,
-        provider: str = DEFAULT_PROVIDER,
-        api_token: Optional[str] = None,
+        llm_config: "LLMConfig" = None,
         instruction: str = None,
         chunk_token_threshold: int = int(1e9),
         overlap_rate: float = OVERLAP_RATE,
         word_token_rate: float = WORD_TOKEN_RATE,
+        # char_token_rate: float = WORD_TOKEN_RATE * 5,
+        # chunk_mode: str = "char",
+        verbose: bool = False,
+        logger: Optional[AsyncLogger] = None,
+        ignore_cache: bool = True,
+        # Deprecated properties
+        provider: str = DEFAULT_PROVIDER,
+        api_token: Optional[str] = None,
         base_url: Optional[str] = None,
         api_base: Optional[str] = None,
         extra_args: Dict = None,
-        verbose: bool = False,
-        logger: Optional[AsyncLogger] = None,
     ):
         super().__init__(None)
         self.provider = provider
-        self.api_token = (
-            api_token
-            or PROVIDER_MODELS.get(provider, "no-token")
-            or os.getenv("OPENAI_API_KEY")
-        )
+        self.api_token = api_token
+        self.base_url = base_url or api_base
+        self.llm_config = llm_config
         self.instruction = instruction
         self.chunk_token_threshold = chunk_token_threshold
         self.overlap_rate = overlap_rate
-        self.word_token_rate = word_token_rate
-        self.base_url = base_url
-        self.api_base = api_base or base_url
+        self.word_token_rate = word_token_rate or WORD_TOKEN_RATE
+        # self.chunk_mode: str = chunk_mode
+        # self.char_token_rate = char_token_rate or word_token_rate / 5
+        # self.token_rate = word_token_rate if chunk_mode == "word" else self.char_token_rate
+        self.token_rate = word_token_rate or WORD_TOKEN_RATE
         self.extra_args = extra_args or {}
+        self.ignore_cache = ignore_cache
         self.verbose = verbose
-        
+
         # Setup logger with custom styling for LLM operations
         if logger:
             self.logger = logger
         elif verbose:
             self.logger = AsyncLogger(
-                verbose=True,
+                verbose=verbose,
                 icons={
                     **AsyncLogger.DEFAULT_ICONS,
                     "LLM": "★",  # Star for LLM operations
                     "CHUNK": "◈",  # Diamond for chunks
-                    "CACHE": "⚡", # Lightning for cache operations
+                    "CACHE": "⚡",  # Lightning for cache operations
                 },
                 colors={
                     **AsyncLogger.DEFAULT_COLORS,
-                    LogLevel.INFO: Fore.MAGENTA + Style.DIM,  # Dimmed purple for LLM ops
-                }
+                    LogLevel.INFO: Fore.MAGENTA
+                    + Style.DIM,  # Dimmed purple for LLM ops
+                },
             )
         else:
             self.logger = None
-        
+
         self.usages = []
         self.total_usage = TokenUsage()
+    
+    def __setattr__(self, name, value):
+        """Handle attribute setting."""
+        # TODO: Planning to set properties dynamically based on the __init__ signature
+        sig = inspect.signature(self.__init__)
+        all_params = sig.parameters  # Dictionary of parameter names and their details
 
+        if name in self._UNWANTED_PROPS and value is not all_params[name].default:
+            raise AttributeError(f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}")
+        
+        super().__setattr__(name, value)  
+        
     def _get_cache_key(self, html: str, instruction: str) -> str:
         """Generate a unique cache key based on HTML and instruction"""
         content = f"{html}{instruction}"
         return hashlib.md5(content.encode()).hexdigest()
 
     def _merge_chunks(self, text: str) -> List[str]:
-        """Split text into chunks with overlap"""
-        # Calculate tokens and sections
-        total_tokens = len(text.split()) * self.word_token_rate
-        num_sections = max(1, math.floor(total_tokens / self.chunk_token_threshold))
-        adjusted_chunk_threshold = total_tokens / num_sections
+        """Split text into chunks with overlap using char or word mode."""
+        ov = int(self.chunk_token_threshold * self.overlap_rate)
+        sections = merge_chunks(
+            docs=[text],
+            target_size=self.chunk_token_threshold,
+            overlap=ov,
+            word_token_ratio=self.word_token_rate,
+        )
+        return sections
 
-        # Split into words
-        words = text.split()
-        chunks = []
-        current_chunk = []
-        current_token_count = 0
-
-        for word in words:
-            word_tokens = len(word) * self.word_token_rate
-            if current_token_count + word_tokens <= adjusted_chunk_threshold:
-                current_chunk.append(word)
-                current_token_count += word_tokens
-            else:
-                # Add overlap if not the last chunk
-                if chunks and self.overlap_rate > 0:
-                    overlap_size = int(len(current_chunk) * self.overlap_rate)
-                    current_chunk.extend(current_chunk[-overlap_size:])
-                
-                chunks.append(" ".join(current_chunk))
-                current_chunk = [word]
-                current_token_count = word_tokens
-
-        if current_chunk:
-            chunks.append(" ".join(current_chunk))
-
-        return chunks
-
-    def filter_content(self, html: str, ignore_cache: bool = False) -> List[str]:
+    def filter_content(self, html: str, ignore_cache: bool = True) -> List[str]:
         if not html or not isinstance(html, str):
             return []
 
         if self.logger:
             self.logger.info(
-                "Starting LLM content filtering process", 
+                "Starting LLM markdown content filtering process",
                 tag="LLM",
-                params={"provider": self.provider},
-                colors={"provider": Fore.CYAN}
+                params={"provider": self.llm_config.provider},
+                colors={"provider": Fore.CYAN},
             )
 
         # Cache handling
@@ -853,65 +901,88 @@ class LLMContentFilter(RelevantContentFilter):
         cache_key = self._get_cache_key(html, self.instruction or "")
         cache_file = cache_dir / f"{cache_key}.json"
 
+        # if ignore_cache == None:
+        ignore_cache = self.ignore_cache
+
         if not ignore_cache and cache_file.exists():
             if self.logger:
-                self.logger.info("Found cached result", tag="CACHE")
+                self.logger.info("Found  cached markdown result", tag="CACHE")
             try:
-                with cache_file.open('r') as f:
+                with cache_file.open("r") as f:
                     cached_data = json.load(f)
-                    usage = TokenUsage(**cached_data['usage'])
+                    usage = TokenUsage(**cached_data["usage"])
                     self.usages.append(usage)
                     self.total_usage.completion_tokens += usage.completion_tokens
                     self.total_usage.prompt_tokens += usage.prompt_tokens
                     self.total_usage.total_tokens += usage.total_tokens
-                    return cached_data['blocks']
+                    return cached_data["blocks"]
             except Exception as e:
                 if self.logger:
-                    self.logger.error(f"Cache read error: {str(e)}", tag="CACHE")
+                    self.logger.error(
+                        f"LLM markdown: Cache read error: {str(e)}", tag="CACHE"
+                    )
 
         # Split into chunks
         html_chunks = self._merge_chunks(html)
         if self.logger:
             self.logger.info(
-                "Split content into {chunk_count} chunks", 
+                "LLM markdown: Split content into {chunk_count} chunks",
                 tag="CHUNK",
                 params={"chunk_count": len(html_chunks)},
-                colors={"chunk_count": Fore.YELLOW}
+                colors={"chunk_count": Fore.YELLOW},
             )
-        
-        extracted_content = []
+
         start_time = time.time()
-        
+
         # Process chunks in parallel
         with ThreadPoolExecutor(max_workers=4) as executor:
             futures = []
             for i, chunk in enumerate(html_chunks):
                 if self.logger:
                     self.logger.debug(
-                        "Processing chunk {chunk_num}/{total_chunks}", 
+                        "LLM markdown: Processing chunk {chunk_num}/{total_chunks}",
                         tag="CHUNK",
-                        params={
-                            "chunk_num": i + 1,
-                            "total_chunks": len(html_chunks)
-                        }
+                        params={"chunk_num": i + 1, "total_chunks": len(html_chunks)},
                     )
 
                 prompt_variables = {
                     "HTML": escape_json_string(sanitize_html(chunk)),
-                    "REQUEST": self.instruction or "Convert this HTML into clean, relevant markdown, removing any noise or irrelevant content."
+                    "REQUEST": self.instruction
+                    or "Convert this HTML into clean, relevant markdown, removing any noise or irrelevant content.",
                 }
 
                 prompt = PROMPT_FILTER_CONTENT
                 for var, value in prompt_variables.items():
                     prompt = prompt.replace("{" + var + "}", value)
 
+                def _proceed_with_chunk(
+                    provider: str,
+                    prompt: str,
+                    api_token: str,
+                    base_url: Optional[str] = None,
+                    extra_args: Dict = {},
+                ) -> List[str]:
+                    if self.logger:
+                        self.logger.info(
+                            "LLM Markdown: Processing chunk {chunk_num}",
+                            tag="CHUNK",
+                            params={"chunk_num": i + 1},
+                        )
+                    return perform_completion_with_backoff(
+                        provider,
+                        prompt,
+                        api_token,
+                        base_url=base_url,
+                        extra_args=extra_args,
+                    )
+
                 future = executor.submit(
-                    perform_completion_with_backoff,
-                    self.provider,
+                    _proceed_with_chunk,
+                    self.llm_config.provider,
                     prompt,
-                    self.api_token,
-                    base_url=self.api_base,
-                    extra_args=self.extra_args
+                    self.llm_config.api_token,
+                    self.llm_config.base_url,
+                    self.extra_args,
                 )
                 futures.append((i, future))
 
@@ -920,59 +991,61 @@ class LLMContentFilter(RelevantContentFilter):
             for i, future in sorted(futures):
                 try:
                     response = future.result()
-                    
+
                     # Track usage
                     usage = TokenUsage(
                         completion_tokens=response.usage.completion_tokens,
                         prompt_tokens=response.usage.prompt_tokens,
                         total_tokens=response.usage.total_tokens,
-                        completion_tokens_details=response.usage.completion_tokens_details.__dict__ 
-                        if response.usage.completion_tokens_details else {},
-                        prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
-                        if response.usage.prompt_tokens_details else {},
+                        completion_tokens_details=(
+                            response.usage.completion_tokens_details.__dict__
+                            if response.usage.completion_tokens_details
+                            else {}
+                        ),
+                        prompt_tokens_details=(
+                            response.usage.prompt_tokens_details.__dict__
+                            if response.usage.prompt_tokens_details
+                            else {}
+                        ),
                     )
                     self.usages.append(usage)
                     self.total_usage.completion_tokens += usage.completion_tokens
                     self.total_usage.prompt_tokens += usage.prompt_tokens
                     self.total_usage.total_tokens += usage.total_tokens
 
-                    blocks = extract_xml_data(["content"], response.choices[0].message.content)["content"]
+                    blocks = extract_xml_data(
+                        ["content"], response.choices[0].message.content
+                    )["content"]
                     if blocks:
                         ordered_results.append(blocks)
                         if self.logger:
                             self.logger.success(
-                                "Successfully processed chunk {chunk_num}", 
+                                "LLM markdown: Successfully processed chunk {chunk_num}",
                                 tag="CHUNK",
-                                params={"chunk_num": i + 1}
+                                params={"chunk_num": i + 1},
                             )
                 except Exception as e:
                     if self.logger:
                         self.logger.error(
-                            "Error processing chunk {chunk_num}: {error}", 
+                            "LLM markdown: Error processing chunk {chunk_num}: {error}",
                             tag="CHUNK",
-                            params={
-                                "chunk_num": i + 1,
-                                "error": str(e)
-                            }
+                            params={"chunk_num": i + 1, "error": str(e)},
                         )
 
         end_time = time.time()
         if self.logger:
             self.logger.success(
-                "Completed processing in {time:.2f}s", 
+                "LLM markdown: Completed processing in {time:.2f}s",
                 tag="LLM",
                 params={"time": end_time - start_time},
-                colors={"time": Fore.YELLOW}
+                colors={"time": Fore.YELLOW},
             )
 
         result = ordered_results if ordered_results else []
 
         # Cache the final result
-        cache_data = {
-            'blocks': result,
-            'usage': self.total_usage.__dict__
-        }
-        with cache_file.open('w') as f:
+        cache_data = {"blocks": result, "usage": self.total_usage.__dict__}
+        with cache_file.open("w") as f:
             json.dump(cache_data, f)
             if self.logger:
                 self.logger.info("Cached results for future use", tag="CACHE")
@@ -996,4 +1069,4 @@ class LLMContentFilter(RelevantContentFilter):
                 print(
                     f"{i:<10} {usage.completion_tokens:>12,} "
                     f"{usage.prompt_tokens:>12,} {usage.total_tokens:>12,}"
-                )
+                )

crawl4ai/content_scraping_strategy.py

@@ -155,6 +155,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
                 for aud in raw_result.get("media", {}).get("audios", [])
                 if aud
             ],
+            tables=raw_result.get("media", {}).get("tables", [])
         )
 
         # Convert links
@@ -193,6 +194,153 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         """
         return await asyncio.to_thread(self._scrap, url, html, **kwargs)
 
+    def is_data_table(self, table: Tag, **kwargs) -> bool:
+        """
+        Determine if a table element is a data table (not a layout table).
+
+        Args:
+            table (Tag): BeautifulSoup Tag representing a table element
+            **kwargs: Additional keyword arguments including table_score_threshold
+
+        Returns:
+            bool: True if the table is a data table, False otherwise
+        """
+        score = 0
+        
+        # Check for thead and tbody
+        has_thead = len(table.select('thead')) > 0
+        has_tbody = len(table.select('tbody')) > 0
+        if has_thead:
+            score += 2
+        if has_tbody:
+            score += 1
+            
+        # Check for th elements
+        th_count = len(table.select('th'))
+        if th_count > 0:
+            score += 2
+            if has_thead or len(table.select('tr:first-child th')) > 0:
+                score += 1
+                
+        # Check for nested tables
+        if len(table.select('table')) > 0:
+            score -= 3
+            
+        # Role attribute check
+        role = table.get('role', '').lower()
+        if role in {'presentation', 'none'}:
+            score -= 3
+            
+        # Column consistency
+        rows = table.select('tr')
+        if not rows:
+            return False
+            
+        col_counts = [len(row.select('td, th')) for row in rows]
+        avg_cols = sum(col_counts) / len(col_counts)
+        variance = sum((c - avg_cols)**2 for c in col_counts) / len(col_counts)
+        if variance < 1:
+            score += 2
+            
+        # Caption and summary
+        if table.select('caption'):
+            score += 2
+        if table.has_attr('summary') and table['summary']:
+            score += 1
+            
+        # Text density
+        total_text = sum(len(cell.get_text().strip()) for row in rows for cell in row.select('td, th'))
+        total_tags = sum(1 for _ in table.descendants if isinstance(_, Tag))
+        text_ratio = total_text / (total_tags + 1e-5)
+        if text_ratio > 20:
+            score += 3
+        elif text_ratio > 10:
+            score += 2
+            
+        # Data attributes
+        data_attrs = sum(1 for attr in table.attrs if attr.startswith('data-'))
+        score += data_attrs * 0.5
+        
+        # Size check
+        if avg_cols >= 2 and len(rows) >= 2:
+            score += 2
+            
+        threshold = kwargs.get('table_score_threshold', 7)
+        return score >= threshold
+    
+    def extract_table_data(self, table: Tag) -> dict:
+        """
+        Extract structured data from a table element.
+        
+        Args:
+            table (Tag): BeautifulSoup Tag representing a table element
+            
+        Returns:
+            dict: Dictionary containing table data (headers, rows, caption, summary)
+        """
+        caption_elem = table.select_one('caption')
+        caption = caption_elem.get_text().strip() if caption_elem else ""
+        summary = table.get('summary', '').strip()
+        
+        # Extract headers with colspan handling
+        headers = []
+        thead_rows = table.select('thead tr')
+        if thead_rows:
+            header_cells = thead_rows[0].select('th')
+            for cell in header_cells:
+                text = cell.get_text().strip()
+                colspan = int(cell.get('colspan', 1))
+                headers.extend([text] * colspan)
+        else:
+            first_row = table.select('tr:first-child')
+            if first_row:
+                for cell in first_row[0].select('th, td'):
+                    text = cell.get_text().strip()
+                    colspan = int(cell.get('colspan', 1))
+                    headers.extend([text] * colspan)
+        
+        # Extract rows with colspan handling
+        rows = []
+        all_rows = table.select('tr')
+        thead = table.select_one('thead')
+        tbody_rows = []
+
+        if thead:
+            thead_rows = thead.select('tr')
+            tbody_rows = [row for row in all_rows if row not in thead_rows]
+        else:
+            if all_rows and all_rows[0].select('th'):
+                tbody_rows = all_rows[1:]
+            else:
+                tbody_rows = all_rows
+                
+        for row in tbody_rows:        
+        # for row in table.select('tr:not(:has(ancestor::thead))'):
+            row_data = []
+            for cell in row.select('td'):
+                text = cell.get_text().strip()
+                colspan = int(cell.get('colspan', 1))
+                row_data.extend([text] * colspan)
+            if row_data:
+                rows.append(row_data)
+                
+        # Align rows with headers
+        max_columns = len(headers) if headers else (max(len(row) for row in rows) if rows else 0)
+        aligned_rows = []
+        for row in rows:
+            aligned = row[:max_columns] + [''] * (max_columns - len(row))
+            aligned_rows.append(aligned)
+            
+        if not headers:
+            headers = [f"Column {i+1}" for i in range(max_columns)]
+            
+        return {
+            "headers": headers,
+            "rows": aligned_rows,
+            "caption": caption,
+            "summary": summary,
+        }
+    
     def flatten_nested_elements(self, node):
         """
         Flatten nested elements in a HTML tree.
@@ -431,7 +579,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         Returns:
             dict: A dictionary containing the processed element information.
         """
-        media = {"images": [], "videos": [], "audios": []}
+        media = {"images": [], "videos": [], "audios": [], "tables": []}
         internal_links_dict = {}
         external_links_dict = {}
         self._process_element(
@@ -529,6 +677,9 @@ class WebScrapingStrategy(ContentScrapingStrategy):
                         if normalized_href not in external_links_dict:
                             external_links_dict[normalized_href] = link_data
                     else:
+                        if kwargs.get("exclude_internal_links", False):
+                            element.decompose()
+                            return False
                         if normalized_href not in internal_links_dict:
                             internal_links_dict[normalized_href] = link_data
 
@@ -629,7 +780,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
 
             try:
                 self.remove_unwanted_attributes(
-                    element, IMPORTANT_ATTRS, kwargs.get("keep_data_attributes", False)
+                    element, IMPORTANT_ATTRS + kwargs.get("keep_attrs", []) , kwargs.get("keep_data_attributes", False)
                 )
             except Exception as e:
                 # print('Error removing unwanted attributes:', str(e))
@@ -685,6 +836,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         html: str,
         word_count_threshold: int = MIN_WORD_THRESHOLD,
         css_selector: str = None,
+        target_elements: List[str] = None,
         **kwargs,
     ) -> Dict[str, Any]:
         """
@@ -739,22 +891,37 @@ class WebScrapingStrategy(ContentScrapingStrategy):
                 for element in body.select(excluded_selector):
                     element.extract()
 
-        if css_selector:
-            selected_elements = body.select(css_selector)
-            if not selected_elements:
-                return {
-                    "markdown": "",
-                    "cleaned_html": "",
-                    "success": True,
-                    "media": {"images": [], "videos": [], "audios": []},
-                    "links": {"internal": [], "external": []},
-                    "metadata": {},
-                    "message": f"No elements found for CSS selector: {css_selector}",
-                }
-                # raise InvalidCSSSelectorError(f"Invalid CSS selector, No elements found for CSS selector: {css_selector}")
-            body = soup.new_tag("div")
-            for el in selected_elements:
-                body.append(el)
+        # if False and css_selector:
+        #     selected_elements = body.select(css_selector)
+        #     if not selected_elements:
+        #         return {
+        #             "markdown": "",
+        #             "cleaned_html": "",
+        #             "success": True,
+        #             "media": {"images": [], "videos": [], "audios": []},
+        #             "links": {"internal": [], "external": []},
+        #             "metadata": {},
+        #             "message": f"No elements found for CSS selector: {css_selector}",
+        #         }
+        #         # raise InvalidCSSSelectorError(f"Invalid CSS selector, No elements found for CSS selector: {css_selector}")
+        #     body = soup.new_tag("div")
+        #     for el in selected_elements:
+        #         body.append(el)
+
+        content_element = None
+        if target_elements:
+            try:
+                for_content_targeted_element = []
+                for target_element in target_elements:
+                    for_content_targeted_element.extend(body.select(target_element))
+                content_element = soup.new_tag("div")
+                for el in for_content_targeted_element:
+                    content_element.append(el)
+            except Exception as e:
+                self._log("error", f"Error with target element detection: {str(e)}", "SCRAPE")
+                return None
+        else:
+            content_element = body        
 
         kwargs["exclude_social_media_domains"] = set(
             kwargs.get("exclude_social_media_domains", []) + SOCIAL_MEDIA_DOMAINS
@@ -794,6 +961,15 @@ class WebScrapingStrategy(ContentScrapingStrategy):
             if result is not None
             for img in result
         ]
+        
+        # Process tables if not excluded
+        excluded_tags = set(kwargs.get("excluded_tags", []) or [])
+        if 'table' not in excluded_tags:
+            tables = body.find_all('table')
+            for table in tables:
+                if self.is_data_table(table, **kwargs):
+                    table_data = self.extract_table_data(table)
+                    media["tables"].append(table_data)
 
         body = self.flatten_nested_elements(body)
         base64_pattern = re.compile(r'data:image/[^;]+;base64,([^"]+)')
@@ -805,7 +981,7 @@ class WebScrapingStrategy(ContentScrapingStrategy):
 
         str_body = ""
         try:
-            str_body = body.encode_contents().decode("utf-8")
+            str_body = content_element.encode_contents().decode("utf-8")
         except Exception:
             # Reset body to the original HTML
             success = False
@@ -844,7 +1020,6 @@ class WebScrapingStrategy(ContentScrapingStrategy):
         cleaned_html = str_body.replace("\n\n", "\n").replace("  ", " ")
 
         return {
-            # **markdown_content,
             "cleaned_html": cleaned_html,
             "success": success,
             "media": media,
@@ -1184,12 +1359,125 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
 
         return root
 
+    def is_data_table(self, table: etree.Element, **kwargs) -> bool:
+        score = 0
+        # Check for thead and tbody
+        has_thead = len(table.xpath(".//thead")) > 0
+        has_tbody = len(table.xpath(".//tbody")) > 0
+        if has_thead:
+            score += 2
+        if has_tbody:
+            score += 1
+
+        # Check for th elements
+        th_count = len(table.xpath(".//th"))
+        if th_count > 0:
+            score += 2
+            if has_thead or table.xpath(".//tr[1]/th"):
+                score += 1
+
+        # Check for nested tables
+        if len(table.xpath(".//table")) > 0:
+            score -= 3
+
+        # Role attribute check
+        role = table.get("role", "").lower()
+        if role in {"presentation", "none"}:
+            score -= 3
+
+        # Column consistency
+        rows = table.xpath(".//tr")
+        if not rows:
+            return False
+        col_counts = [len(row.xpath(".//td|.//th")) for row in rows]
+        avg_cols = sum(col_counts) / len(col_counts)
+        variance = sum((c - avg_cols)**2 for c in col_counts) / len(col_counts)
+        if variance < 1:
+            score += 2
+
+        # Caption and summary
+        if table.xpath(".//caption"):
+            score += 2
+        if table.get("summary"):
+            score += 1
+
+        # Text density
+        total_text = sum(len(''.join(cell.itertext()).strip()) for row in rows for cell in row.xpath(".//td|.//th"))
+        total_tags = sum(1 for _ in table.iterdescendants())
+        text_ratio = total_text / (total_tags + 1e-5)
+        if text_ratio > 20:
+            score += 3
+        elif text_ratio > 10:
+            score += 2
+
+        # Data attributes
+        data_attrs = sum(1 for attr in table.attrib if attr.startswith('data-'))
+        score += data_attrs * 0.5
+
+        # Size check
+        if avg_cols >= 2 and len(rows) >= 2:
+            score += 2
+
+        threshold = kwargs.get("table_score_threshold", 7)
+        return score >= threshold
+
+    def extract_table_data(self, table: etree.Element) -> dict:
+        caption = table.xpath(".//caption/text()")
+        caption = caption[0].strip() if caption else ""
+        summary = table.get("summary", "").strip()
+
+        # Extract headers with colspan handling
+        headers = []
+        thead_rows = table.xpath(".//thead/tr")
+        if thead_rows:
+            header_cells = thead_rows[0].xpath(".//th")
+            for cell in header_cells:
+                text = cell.text_content().strip()
+                colspan = int(cell.get("colspan", 1))
+                headers.extend([text] * colspan)
+        else:
+            first_row = table.xpath(".//tr[1]")
+            if first_row:
+                for cell in first_row[0].xpath(".//th|.//td"):
+                    text = cell.text_content().strip()
+                    colspan = int(cell.get("colspan", 1))
+                    headers.extend([text] * colspan)
+
+        # Extract rows with colspan handling
+        rows = []
+        for row in table.xpath(".//tr[not(ancestor::thead)]"):
+            row_data = []
+            for cell in row.xpath(".//td"):
+                text = cell.text_content().strip()
+                colspan = int(cell.get("colspan", 1))
+                row_data.extend([text] * colspan)
+            if row_data:
+                rows.append(row_data)
+
+        # Align rows with headers
+        max_columns = len(headers) if headers else (max(len(row) for row in rows) if rows else 0)
+        aligned_rows = []
+        for row in rows:
+            aligned = row[:max_columns] + [''] * (max_columns - len(row))
+            aligned_rows.append(aligned)
+
+        if not headers:
+            headers = [f"Column {i+1}" for i in range(max_columns)]
+
+        return {
+            "headers": headers,
+            "rows": aligned_rows,
+            "caption": caption,
+            "summary": summary,
+        }
+
     def _scrap(
         self,
         url: str,
         html: str,
         word_count_threshold: int = MIN_WORD_THRESHOLD,
         css_selector: str = None,
+        target_elements: List[str] = None,
         **kwargs,
     ) -> Dict[str, Any]:
         if not html:
@@ -1240,24 +1528,38 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
                 meta = {}
 
             # Handle CSS selector targeting
-            if css_selector:
+            # if css_selector:
+            #     try:
+            #         selected_elements = body.cssselect(css_selector)
+            #         if not selected_elements:
+            #             return {
+            #                 "markdown": "",
+            #                 "cleaned_html": "",
+            #                 "success": True,
+            #                 "media": {"images": [], "videos": [], "audios": []},
+            #                 "links": {"internal": [], "external": []},
+            #                 "metadata": meta,
+            #                 "message": f"No elements found for CSS selector: {css_selector}",
+            #             }
+            #         body = lhtml.Element("div")
+            #         body.extend(selected_elements)
+            #     except Exception as e:
+            #         self._log("error", f"Error with CSS selector: {str(e)}", "SCRAPE")
+            #         return None
+
+            content_element = None
+            if target_elements:
                 try:
-                    selected_elements = body.cssselect(css_selector)
-                    if not selected_elements:
-                        return {
-                            "markdown": "",
-                            "cleaned_html": "",
-                            "success": True,
-                            "media": {"images": [], "videos": [], "audios": []},
-                            "links": {"internal": [], "external": []},
-                            "metadata": meta,
-                            "message": f"No elements found for CSS selector: {css_selector}",
-                        }
-                    body = lhtml.Element("div")
-                    body.extend(selected_elements)
+                    for_content_targeted_element = []
+                    for target_element in target_elements:
+                        for_content_targeted_element.extend(body.cssselect(target_element))
+                    content_element = lhtml.Element("div")
+                    content_element.extend(for_content_targeted_element)
                 except Exception as e:
-                    self._log("error", f"Error with CSS selector: {str(e)}", "SCRAPE")
+                    self._log("error", f"Error with target element detection: {str(e)}", "SCRAPE")
                     return None
+            else:
+                content_element = body
 
             # Remove script and style tags
             for tag in ["script", "style", "link", "meta", "noscript"]:
@@ -1281,7 +1583,7 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
                         form.getparent().remove(form)
 
             # Process content
-            media = {"images": [], "videos": [], "audios": []}
+            media = {"images": [], "videos": [], "audios": [], "tables": []}
             internal_links_dict = {}
             external_links_dict = {}
 
@@ -1295,6 +1597,13 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
                 **kwargs,
             )
 
+            if 'table' not in excluded_tags:
+                tables = body.xpath(".//table")
+                for table in tables:
+                    if self.is_data_table(table, **kwargs):
+                        table_data = self.extract_table_data(table)
+                        media["tables"].append(table_data)
+
             # Handle only_text option
             if kwargs.get("only_text", False):
                 for tag in ONLY_TEXT_ELIGIBLE_TAGS:
@@ -1321,7 +1630,8 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
 
             # Generate output HTML
             cleaned_html = lhtml.tostring(
-                body,
+                # body,   
+                content_element,
                 encoding="unicode",
                 pretty_print=True,
                 method="html",
@@ -1366,7 +1676,12 @@ class LXMLWebScrapingStrategy(WebScrapingStrategy):
             return {
                 "cleaned_html": cleaned_html,
                 "success": False,
-                "media": {"images": [], "videos": [], "audios": []},
+                "media": {
+                    "images": [],
+                    "videos": [],
+                    "audios": [],
+                    "tables": []
+                },
                 "links": {"internal": [], "external": []},
                 "metadata": {},
             }

crawl4ai/crawlers/amazon_product/crawler.py

@@ -0,0 +1,20 @@
+from crawl4ai.hub import BaseCrawler
+
+__meta__ = {
+    "version": "1.2.0",
+    "tested_on": ["amazon.com"],
+    "rate_limit": "50 RPM",
+    "schema": {"product": ["name", "price"]}
+}
+
+class AmazonProductCrawler(BaseCrawler):
+    async def run(self, url: str, **kwargs) -> str:
+        try:
+            self.logger.info(f"Crawling {url}")
+            return '{"product": {"name": "Test Amazon Product"}}'
+        except Exception as e:
+            self.logger.error(f"Crawl failed: {str(e)}")
+            return json.dumps({
+                "error": str(e),
+                "metadata": self.meta  # Include meta in error response
+            })            

crawl4ai/crawlers/google_search/crawler.py

@@ -0,0 +1,131 @@
+from crawl4ai import BrowserConfig, AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai.hub import BaseCrawler
+from crawl4ai.utils import optimize_html, get_home_folder, preprocess_html_for_schema
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from pathlib import Path
+import json
+import os
+from typing import Dict
+
+
+class GoogleSearchCrawler(BaseCrawler):
+    __meta__ = {
+        "version": "1.0.0",
+        "tested_on": ["google.com/search*"],
+        "rate_limit": "10 RPM",
+        "description": "Crawls Google Search results (text + images)",
+    }
+
+    def __init__(self):
+        super().__init__()
+        self.js_script = (Path(__file__).parent /
+                          "script.js").read_text()
+
+    async def run(self, url="", query: str = "", search_type: str = "text", schema_cache_path = None, **kwargs) -> str:
+        """Crawl Google Search results for a query"""
+        url = f"https://www.google.com/search?q={query}&gl=sg&hl=en" if search_type == "text" else f"https://www.google.com/search?q={query}&gl=sg&hl=en&tbs=qdr:d&udm=2"
+        if kwargs.get("page_start", 1) > 1:
+            url = f"{url}&start={kwargs['page_start'] * 10}"
+        if kwargs.get("page_length", 1) > 1:
+            url = f"{url}&num={kwargs['page_length']}"
+            
+        browser_config = BrowserConfig(headless=True, verbose=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            config = CrawlerRunConfig(
+                cache_mode=kwargs.get("cache_mode", CacheMode.BYPASS),
+                keep_attrs=["id", "class"],
+                keep_data_attributes=True,
+                delay_before_return_html=kwargs.get(
+                    "delay", 2 if search_type == "image" else 1),
+                js_code=self.js_script if search_type == "image" else None,
+            )
+
+            result = await crawler.arun(url=url, config=config)
+            if not result.success:
+                return json.dumps({"error": result.error})
+
+            if search_type == "image":
+                if result.js_execution_result.get("success", False) is False:
+                    return json.dumps({"error": result.js_execution_result.get("error", "Unknown error")})
+                if "results" in result.js_execution_result:
+                    image_result = result.js_execution_result['results'][0]
+                    if image_result.get("success", False) is False:
+                        return json.dumps({"error": image_result.get("error", "Unknown error")})
+                    return json.dumps(image_result["result"], indent=4)
+
+            # For text search, extract structured data
+            schemas = await self._build_schemas(result.cleaned_html, schema_cache_path)
+            extracted = {
+                key: JsonCssExtractionStrategy(schema=schemas[key]).run(
+                    url=url, sections=[result.html]
+                )
+                for key in schemas
+            }
+            return json.dumps(extracted, indent=4)
+
+    async def _build_schemas(self, html: str, schema_cache_path: str = None) -> Dict[str, Dict]:
+        """Build extraction schemas (organic, top stories, etc.)"""
+        home_dir = get_home_folder() if not schema_cache_path else schema_cache_path
+        os.makedirs(f"{home_dir}/schema", exist_ok=True)
+
+        # cleaned_html = optimize_html(html, threshold=100)
+        cleaned_html = preprocess_html_for_schema(html) 
+
+        organic_schema = None
+        if os.path.exists(f"{home_dir}/schema/organic_schema.json"):
+            with open(f"{home_dir}/schema/organic_schema.json", "r") as f:
+                organic_schema = json.load(f)
+        else:
+            organic_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "snippet": "...",
+            "date": "1 hour ago",
+        }""",
+                query="""The given html is the crawled html from Google search result. Please find the schema for organic search item in the given html, I am interested in title, link, snippet text. date."""
+            )
+
+            with open(f"{home_dir}/schema/organic_schema.json", "w") as f:
+                f.write(json.dumps(organic_schema))
+
+        top_stories_schema = None
+        if os.path.exists(f"{home_dir}/schema/top_stories_schema.json"):
+            with open(f"{home_dir}/schema/top_stories_schema.json", "r") as f:
+                top_stories_schema = json.load(f)
+        else:
+            top_stories_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "source": "Insider Monkey",
+            "date": "1 hour ago",
+        }""",
+                query="""The given html is the crawled html from Google search result. Please find the schema for Top Story item int he given html, I am interested in title, link, source. date and imageUrl."""
+            )
+
+            with open(f"{home_dir}/schema/top_stories_schema.json", "w") as f:
+                f.write(json.dumps(top_stories_schema))
+
+        suggested_query_schema = None
+        if os.path.exists(f"{home_dir}/schema/suggested_query_schema.json"):
+            with open(f"{home_dir}/schema/suggested_query_schema.json", "r") as f:
+                suggested_query_schema = json.load(f)
+        else:
+            suggested_query_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "query": "A for Apple",
+        }""",
+                query="""The given HTML contains the crawled HTML from Google search results. Please find the schema for each suggested query in the section "People also search for" within the given HTML. I am interested in the queries only."""
+            )
+            with open(f"{home_dir}/schema/suggested_query_schema.json", "w") as f:
+                f.write(json.dumps(suggested_query_schema))
+
+        return {
+            "organic_schema": organic_schema,
+            "top_stories_schema": top_stories_schema,
+            "suggested_query_schema": suggested_query_schema,
+        }

crawl4ai/crawlers/google_search/script.js

@@ -0,0 +1,115 @@
+(() => {
+    // Function to extract image data from Google Images page
+    function extractImageData() {
+        const keys = Object.keys(window.W_jd);
+        let allImageData = [];
+        let currentPosition = 0;
+
+        // Get the symbol we'll use (from first valid entry)
+        let targetSymbol;
+        for (let key of keys) {
+            try {
+                const symbols = Object.getOwnPropertySymbols(window.W_jd[key]);
+                if (symbols.length > 0) {
+                    targetSymbol = symbols[0];
+                    break;
+                }
+            } catch (e) {
+                continue;
+            }
+        }
+
+        if (!targetSymbol) return [];
+
+        // Iterate through ALL keys
+        for (let key of keys) {
+            try {
+                const o1 = window.W_jd[key][targetSymbol]
+                if (!o1) continue;
+                const data = Object.values(o1)[0]
+                // const data = window.W_jd[key][targetSymbol]?.Ws;
+                // Check if this is a valid image data entry
+                if (data && Array.isArray(data[1])) {
+                    const processedData = processImageEntry(data, currentPosition);
+                    if (processedData) {
+                        allImageData.push(processedData);
+                        currentPosition++;
+                    }
+                }
+            } catch (e) {
+                continue;
+            }
+        }
+
+        return allImageData;
+    }
+
+    function processImageEntry(entry, position) {
+        const imageData = entry[1];
+        if (!Array.isArray(imageData)) return null;
+
+        // Extract the image ID
+        const imageId = imageData[1];
+        if (!imageId) return null;
+
+        // Find the corresponding DOM element
+        const domElement = document.querySelector(`[data-docid="${imageId}"]`);
+        if (!domElement) return null;
+
+        // Extract data from the array structure
+        const [
+            _,
+            id,
+            thumbnailInfo,
+            imageInfo,
+            __,
+            ___,
+            rgb,
+            ____,
+            _____,
+            metadata
+        ] = imageData;
+
+        // Ensure we have the required data
+        if (!thumbnailInfo || !imageInfo) return null;
+
+        // Extract metadata from DOM
+        const title = domElement?.querySelector('.toI8Rb')?.textContent?.trim();
+        const source = domElement?.querySelector('.guK3rf')?.textContent?.trim();
+        const link = domElement?.querySelector('a.EZAeBe')?.href;
+
+        if (!link) return null;
+
+        // Build Google Image URL
+        const googleUrl = buildGoogleImageUrl(imageInfo[0], link, imageId, imageInfo[1], imageInfo[2]);
+
+        return {
+            title,
+            imageUrl: imageInfo[0],
+            imageWidth: imageInfo[2],
+            imageHeight: imageInfo[1],
+            thumbnailUrl: thumbnailInfo[0],
+            thumbnailWidth: thumbnailInfo[2],
+            thumbnailHeight: thumbnailInfo[1],
+            source,
+            domain: metadata['2000']?.[1] || new URL(link).hostname,
+            link,
+            googleUrl,
+            position: position + 1
+        };
+    }
+
+    function buildGoogleImageUrl(imgUrl, refUrl, tbnid, height, width) {
+        const params = new URLSearchParams({
+            imgurl: imgUrl,
+            tbnid: tbnid,
+            imgrefurl: refUrl,
+            docid: tbnid,
+            w: width.toString(),
+            h: height.toString(),
+        });
+
+        return `https://www.google.com/imgres?${params.toString()}`;
+    }
+    return extractImageData();
+})();

crawl4ai/deep_crawling/__init__.py

@@ -0,0 +1,47 @@
+# deep_crawling/__init__.py
+from .base_strategy import DeepCrawlDecorator, DeepCrawlStrategy
+from .bfs_strategy import BFSDeepCrawlStrategy
+from .bff_strategy import BestFirstCrawlingStrategy
+from .dfs_strategy import DFSDeepCrawlStrategy
+from .filters import (
+    FilterChain,
+    ContentTypeFilter,
+    DomainFilter,
+    URLFilter,
+    URLPatternFilter,
+    FilterStats,
+    ContentRelevanceFilter,
+    SEOFilter
+)
+from .scorers import (
+    KeywordRelevanceScorer,
+    URLScorer,
+    CompositeScorer,
+    DomainAuthorityScorer,
+    FreshnessScorer,
+    PathDepthScorer,
+    ContentTypeScorer
+)
+
+__all__ = [
+    "DeepCrawlDecorator",
+    "DeepCrawlStrategy",
+    "BFSDeepCrawlStrategy",
+    "BestFirstCrawlingStrategy",
+    "DFSDeepCrawlStrategy",
+    "FilterChain",
+    "ContentTypeFilter",
+    "DomainFilter",
+    "URLFilter",
+    "URLPatternFilter",
+    "FilterStats",
+    "ContentRelevanceFilter",
+    "SEOFilter",
+    "KeywordRelevanceScorer",
+    "URLScorer",
+    "CompositeScorer",
+    "DomainAuthorityScorer",
+    "FreshnessScorer",
+    "PathDepthScorer",
+    "ContentTypeScorer",
+]

crawl4ai/deep_crawling/base_strategy.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator, Optional, Set, List, Dict
+from functools import wraps
+from contextvars import ContextVar
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+
+
+class DeepCrawlDecorator:
+    """Decorator that adds deep crawling capability to arun method."""
+    deep_crawl_active = ContextVar("deep_crawl_active", default=False)
+    
+    def __init__(self, crawler: AsyncWebCrawler): 
+        self.crawler = crawler
+
+    def __call__(self, original_arun):
+        @wraps(original_arun)
+        async def wrapped_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
+            # If deep crawling is already active, call the original method to avoid recursion.
+            if config and config.deep_crawl_strategy and not self.deep_crawl_active.get():
+                token = self.deep_crawl_active.set(True)
+                # Await the arun call to get the actual result object.
+                result_obj = await config.deep_crawl_strategy.arun(
+                    crawler=self.crawler,
+                    start_url=url,
+                    config=config
+                )
+                if config.stream:
+                    async def result_wrapper():
+                        try:
+                            async for result in result_obj:
+                                yield result
+                        finally:
+                            self.deep_crawl_active.reset(token)
+                    return result_wrapper()
+                else:
+                    try:
+                        return result_obj
+                    finally:
+                        self.deep_crawl_active.reset(token)
+            return await original_arun(url, config=config, **kwargs)
+        return wrapped_arun
+
+class DeepCrawlStrategy(ABC):
+    """
+    Abstract base class for deep crawling strategies.
+    
+    Core functions:
+      - arun: Main entry point that returns an async generator of CrawlResults.
+      - shutdown: Clean up resources.
+      - can_process_url: Validate a URL and decide whether to process it.
+      - _process_links: Extract and process links from a CrawlResult.
+    """
+
+    @abstractmethod
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) mode:
+        Processes one BFS level at a time, then yields all the results.
+        """
+        pass
+
+    @abstractmethod
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming mode:
+        Processes one BFS level at a time and yields results immediately as they arrive.
+        """
+        pass
+    
+    async def arun(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: Optional[CrawlerRunConfig] = None,
+    ) -> RunManyReturn:
+        """
+        Traverse the given URL using the specified crawler.
+        
+        Args:
+            start_url (str): The URL from which to start crawling.
+            crawler (AsyncWebCrawler): The crawler instance to use.
+            crawler_run_config (Optional[CrawlerRunConfig]): Crawler configuration.
+        
+        Returns:
+            Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+        """
+        if config is None:
+            raise ValueError("CrawlerRunConfig must be provided")
+
+        if config.stream:
+            return self._arun_stream(start_url, crawler, config)
+        else:
+            return await self._arun_batch(start_url, crawler, config)
+
+    def __call__(self, start_url: str, crawler: AsyncWebCrawler, config: CrawlerRunConfig):
+        return self.arun(start_url, crawler, config)
+
+    @abstractmethod
+    async def shutdown(self) -> None:
+        """
+        Clean up resources used by the deep crawl strategy.
+        """
+        pass
+
+    @abstractmethod
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validate the URL format and apply custom filtering logic.
+        
+        Args:
+            url (str): The URL to validate.
+            depth (int): The current depth in the crawl.
+        
+        Returns:
+            bool: True if the URL should be processed, False otherwise.
+        """
+        pass
+
+    @abstractmethod
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_level: List[tuple],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extract and process links from the given crawl result.
+        
+        This method should:
+          - Validate each extracted URL using can_process_url.
+          - Optionally score URLs.
+          - Append valid URLs (and their parent references) to the next_level list.
+          - Update the depths dictionary with the new depth for each URL.
+        
+        Args:
+            result (CrawlResult): The result from a crawl operation.
+            source_url (str): The URL from which this result was obtained.
+            current_depth (int): The depth at which the source URL was processed.
+            visited (Set[str]): Set of already visited URLs.
+            next_level (List[tuple]): List of tuples (url, parent_url) for the next BFS level.
+            depths (Dict[str, int]): Mapping of URLs to their current depth.
+        """
+        pass
+

crawl4ai/deep_crawling/bff_strategy.py

@@ -0,0 +1,255 @@
+# best_first_crawling_strategy.py
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+from urllib.parse import urlparse
+
+from ..models import TraversalStats
+from .filters import FilterChain
+from .scorers import URLScorer
+from . import DeepCrawlStrategy
+
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+
+from math import inf as infinity
+
+# Configurable batch size for processing items from the priority queue
+BATCH_SIZE = 10
+
+
+class BestFirstCrawlingStrategy(DeepCrawlStrategy):
+    """
+    Best-First Crawling Strategy using a priority queue.
+    
+    This strategy prioritizes URLs based on their score, ensuring that higher-value
+    pages are crawled first. It reimplements the core traversal loop to use a priority
+    queue while keeping URL validation and link discovery consistent with our design.
+    
+    Core methods:
+      - arun: Returns either a list (batch mode) or an async generator (stream mode).
+      - _arun_best_first: Core generator that uses a priority queue to yield CrawlResults.
+      - can_process_url: Validates URLs and applies filtering (inherited behavior).
+      - link_discovery: Extracts and validates links from a CrawlResult.
+    """
+    def __init__(
+        self,
+        max_depth: int,
+        filter_chain: FilterChain = FilterChain(),
+        url_scorer: Optional[URLScorer] = None,
+        include_external: bool = False,
+        max_pages: int = infinity,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.url_scorer = url_scorer
+        self.include_external = include_external
+        self.max_pages = max_pages
+        self.logger = logger or logging.getLogger(__name__)
+        self.stats = TraversalStats(start_time=datetime.now())
+        self._cancel_event = asyncio.Event()
+        self._pages_crawled = 0
+
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validate the URL format and apply filtering.
+        For the starting URL (depth 0), filtering is bypassed.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError("Missing scheme or netloc")
+            if parsed.scheme not in ("http", "https"):
+                raise ValueError("Invalid scheme")
+            if "." not in parsed.netloc:
+                raise ValueError("Invalid domain")
+        except Exception as e:
+            self.logger.warning(f"Invalid URL: {url}, error: {e}")
+            return False
+
+        if depth != 0 and not await self.filter_chain.apply(url):
+            return False
+
+        return True
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_links: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extract links from the crawl result, validate them, and append new URLs
+        (with their parent references) to next_links.
+        Also updates the depths dictionary.
+        """
+        new_depth = current_depth + 1
+        if new_depth > self.max_depth:
+            return
+            
+        # If we've reached the max pages limit, don't discover new links
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            return
+
+        # Retrieve internal links; include external links if enabled.
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        # If we have more links than remaining capacity, limit how many we'll process
+        valid_links = []
+        for link in links:
+            url = link.get("href")
+            if url in visited:
+                continue
+            if not await self.can_process_url(url, new_depth):
+                self.stats.urls_skipped += 1
+                continue
+                
+            valid_links.append(url)
+            
+        # If we have more valid links than capacity, limit them
+        if len(valid_links) > remaining_capacity:
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
+            
+        # Record the new depths and add to next_links
+        for url in valid_links:
+            depths[url] = new_depth
+            next_links.append((url, source_url))
+
+    async def _arun_best_first(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Core best-first crawl method using a priority queue.
+        
+        The queue items are tuples of (score, depth, url, parent_url). Lower scores
+        are treated as higher priority. URLs are processed in batches for efficiency.
+        """
+        queue: asyncio.PriorityQueue = asyncio.PriorityQueue()
+        # Push the initial URL with score 0 and depth 0.
+        await queue.put((0, 0, start_url, None))
+        visited: Set[str] = set()
+        depths: Dict[str, int] = {start_url: 0}
+
+        while not queue.empty() and not self._cancel_event.is_set():
+            # Stop if we've reached the max pages limit
+            if self._pages_crawled >= self.max_pages:
+                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                break
+                
+            batch: List[Tuple[float, int, str, Optional[str]]] = []
+            # Retrieve up to BATCH_SIZE items from the priority queue.
+            for _ in range(BATCH_SIZE):
+                if queue.empty():
+                    break
+                item = await queue.get()
+                score, depth, url, parent_url = item
+                if url in visited:
+                    continue
+                visited.add(url)
+                batch.append(item)
+
+            if not batch:
+                continue
+
+            # Process the current batch of URLs.
+            urls = [item[2] for item in batch]
+            batch_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=urls, config=batch_config)
+            async for result in stream_gen:
+                result_url = result.url
+                # Find the corresponding tuple from the batch.
+                corresponding = next((item for item in batch if item[2] == result_url), None)
+                if not corresponding:
+                    continue
+                score, depth, url, parent_url = corresponding
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent_url
+                result.metadata["score"] = score
+                
+                # Count only successful crawls toward max_pages limit
+                if result.success:
+                    self._pages_crawled += 1
+                
+                yield result
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Discover new links from this result
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, result_url, depth, visited, new_links, depths)
+                    
+                    for new_url, new_parent in new_links:
+                        new_depth = depths.get(new_url, depth + 1)
+                        new_score = self.url_scorer.score(new_url) if self.url_scorer else 0
+                        await queue.put((new_score, new_depth, new_url, new_parent))
+
+        # End of crawl.
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Best-first crawl in batch mode.
+        
+        Aggregates all CrawlResults into a list.
+        """
+        results: List[CrawlResult] = []
+        async for result in self._arun_best_first(start_url, crawler, config):
+            results.append(result)
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Best-first crawl in streaming mode.
+        
+        Yields CrawlResults as they become available.
+        """
+        async for result in self._arun_best_first(start_url, crawler, config):
+            yield result
+
+    async def arun(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: Optional[CrawlerRunConfig] = None,
+    ) -> "RunManyReturn":
+        """
+        Main entry point for best-first crawling.
+        
+        Returns either a list (batch mode) or an async generator (stream mode)
+        of CrawlResults.
+        """
+        if config is None:
+            raise ValueError("CrawlerRunConfig must be provided")
+        if config.stream:
+            return self._arun_stream(start_url, crawler, config)
+        else:
+            return await self._arun_batch(start_url, crawler, config)
+
+    async def shutdown(self) -> None:
+        """
+        Signal cancellation and clean up resources.
+        """
+        self._cancel_event.set()
+        self.stats.end_time = datetime.now()

crawl4ai/deep_crawling/bfs_strategy.py

@@ -0,0 +1,245 @@
+# bfs_deep_crawl_strategy.py
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+from urllib.parse import urlparse
+
+from ..models import TraversalStats
+from .filters import FilterChain
+from .scorers import URLScorer
+from . import DeepCrawlStrategy  
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
+from ..utils import normalize_url_for_deep_crawl, efficient_normalize_url_for_deep_crawl
+from math import inf as infinity
+
+class BFSDeepCrawlStrategy(DeepCrawlStrategy):
+    """
+    Breadth-First Search deep crawling strategy.
+    
+    Core functions:
+      - arun: Main entry point; splits execution into batch or stream modes.
+      - link_discovery: Extracts, filters, and (if needed) scores the outgoing URLs.
+      - can_process_url: Validates URL format and applies the filter chain.
+    """
+    def __init__(
+        self,
+        max_depth: int,
+        filter_chain: FilterChain = FilterChain(),
+        url_scorer: Optional[URLScorer] = None,        
+        include_external: bool = False,
+        score_threshold: float = -infinity,
+        max_pages: int = infinity,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.url_scorer = url_scorer
+        self.include_external = include_external
+        self.score_threshold = score_threshold
+        self.max_pages = max_pages
+        self.logger = logger or logging.getLogger(__name__)
+        self.stats = TraversalStats(start_time=datetime.now())
+        self._cancel_event = asyncio.Event()
+        self._pages_crawled = 0
+
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validates the URL and applies the filter chain.
+        For the start URL (depth 0) filtering is bypassed.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError("Missing scheme or netloc")
+            if parsed.scheme not in ("http", "https"):
+                raise ValueError("Invalid scheme")
+            if "." not in parsed.netloc:
+                raise ValueError("Invalid domain")
+        except Exception as e:
+            self.logger.warning(f"Invalid URL: {url}, error: {e}")
+            return False
+
+        if depth != 0 and not await self.filter_chain.apply(url):
+            return False
+
+        return True
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_level: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extracts links from the crawl result, validates and scores them, and
+        prepares the next level of URLs.
+        Each valid URL is appended to next_level as a tuple (url, parent_url)
+        and its depth is tracked.
+        """            
+        next_depth = current_depth + 1
+        if next_depth > self.max_depth:
+            return
+
+        # If we've reached the max pages limit, don't discover new links
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            return
+
+        # Get internal links and, if enabled, external links.
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        valid_links = []
+        
+        # First collect all valid links
+        for link in links:
+            url = link.get("href")
+            # Strip URL fragments to avoid duplicate crawling
+            # base_url = url.split('#')[0] if url else url
+            base_url = normalize_url_for_deep_crawl(url, source_url)
+            if base_url in visited:
+                continue
+            if not await self.can_process_url(url, next_depth):
+                self.stats.urls_skipped += 1
+                continue
+
+            # Score the URL if a scorer is provided
+            score = self.url_scorer.score(base_url) if self.url_scorer else 0
+            
+            # Skip URLs with scores below the threshold
+            if score < self.score_threshold:
+                self.logger.debug(f"URL {url} skipped: score {score} below threshold {self.score_threshold}")
+                self.stats.urls_skipped += 1
+                continue
+            
+            valid_links.append((base_url, score))
+        
+        # If we have more valid links than capacity, sort by score and take the top ones
+        if len(valid_links) > remaining_capacity:
+            if self.url_scorer:
+                # Sort by score in descending order
+                valid_links.sort(key=lambda x: x[1], reverse=True)
+            # Take only as many as we have capacity for
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
+            
+        # Process the final selected links
+        for url, score in valid_links:
+            # attach the score to metadata if needed
+            if score:
+                result.metadata = result.metadata or {}
+                result.metadata["score"] = score
+            next_level.append((url, source_url))
+            depths[url] = next_depth
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) mode:
+        Processes one BFS level at a time, then yields all the results.
+        """
+        visited: Set[str] = set()
+        # current_level holds tuples: (url, parent_url)
+        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        results: List[CrawlResult] = []
+
+        while current_level and not self._cancel_event.is_set():
+            next_level: List[Tuple[str, Optional[str]]] = []
+            urls = [url for url, _ in current_level]
+            visited.update(urls)
+
+            # Clone the config to disable deep crawling recursion and enforce batch mode.
+            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
+            batch_results = await crawler.arun_many(urls=urls, config=batch_config)
+            
+            # Update pages crawled counter - count only successful crawls
+            successful_results = [r for r in batch_results if r.success]
+            self._pages_crawled += len(successful_results)
+            
+            for result in batch_results:
+                url = result.url
+                depth = depths.get(url, 0)
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                result.metadata["parent_url"] = parent_url
+                results.append(result)
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Link discovery will handle the max pages limit internally
+                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+
+            current_level = next_level
+
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming mode:
+        Processes one BFS level at a time and yields results immediately as they arrive.
+        """
+        visited: Set[str] = set()
+        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        while current_level and not self._cancel_event.is_set():
+            next_level: List[Tuple[str, Optional[str]]] = []
+            urls = [url for url, _ in current_level]
+            visited.update(urls)
+
+            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=urls, config=stream_config)
+            
+            # Keep track of processed results for this batch
+            results_count = 0
+            async for result in stream_gen:
+                url = result.url
+                depth = depths.get(url, 0)
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                result.metadata["parent_url"] = parent_url
+                
+                # Count only successful crawls
+                if result.success:
+                    self._pages_crawled += 1
+                
+                results_count += 1
+                yield result
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Link discovery will handle the max pages limit internally
+                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+            
+            # If we didn't get results back (e.g. due to errors), avoid getting stuck in an infinite loop
+            # by considering these URLs as visited but not counting them toward the max_pages limit
+            if results_count == 0 and urls:
+                self.logger.warning(f"No results returned for {len(urls)} URLs, marking as visited")
+                
+            current_level = next_level
+
+    async def shutdown(self) -> None:
+        """
+        Clean up resources and signal cancellation of the crawl.
+        """
+        self._cancel_event.set()
+        self.stats.end_time = datetime.now()

crawl4ai/deep_crawling/crazy.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+# I just got crazy, trying to wrute K&R C but in Python. Right now I feel like I'm in a quantum state.
+# I probably won't use this; I just want to leave it here. A century later, the future human race will be like, "WTF?"
+
+# ------ Imports That Will Make You Question Reality ------ #
+from functools import wraps
+from contextvars import ContextVar
+import inspect
+
+from crawl4ai import CacheMode
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai.models import CrawlResult, TraversalStats
+from crawl4ai.deep_crawling.filters import FilterChain
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+import time
+import logging
+from urllib.parse import urlparse
+
+from abc import ABC, abstractmethod
+from collections import deque
+import asyncio
+from typing import (
+    AsyncGenerator,
+    Dict,
+    List,
+    TypeVar,
+    Generic,
+    Tuple,
+    Callable,
+    Awaitable,
+    Union,
+)
+from functools import lru_cache
+import mmh3
+from bitarray import bitarray
+import numpy as np
+from heapq import heappush, heappop
+
+# ------ Type Algebra Mastery ------ #
+CrawlResultT = TypeVar("CrawlResultT", bound="CrawlResult")
+PriorityT = TypeVar("PriorityT")
+P = TypeVar("P")
+
+# ------ Hyperscalar Context Management ------ #
+deep_crawl_ctx = ContextVar("deep_crawl_stack", default=deque())
+
+# ------ Algebraic Crawler Monoid ------ #
+class TraversalContext:
+    __slots__ = ('visited', 'frontier', 'depths', 'priority_fn', 'current_depth')
+    
+    def __init__(self,
+                 priority_fn: Callable[[str], Awaitable[float]] = lambda _: 1.0):
+        self.visited: BloomFilter = BloomFilter(10**6, 0.01)  # 1M items, 1% FP
+        self.frontier: PriorityQueue = PriorityQueue()
+        self.depths: Dict[str, int] = {}
+        self.priority_fn = priority_fn
+        self.current_depth = 0
+
+    def clone_for_level(self) -> TraversalContext:
+        """Monadic context propagation"""
+        new_ctx = TraversalContext(self.priority_fn)
+        new_ctx.visited = self.visited.copy()
+        new_ctx.depths = self.depths.copy()
+        new_ctx.current_depth = self.current_depth
+        return new_ctx
+
+class PriorityQueue(Generic[PriorityT]):
+    """Fibonacci heap-inspired priority queue with O(1) amortized operations"""
+    __slots__ = ('_heap', '_index')
+
+    def __init__(self):
+        self._heap: List[Tuple[PriorityT, float, P]] = []
+        self._index: Dict[P, int] = {}
+
+    def insert(self, priority: PriorityT, item: P) -> None:
+        tiebreaker = time.time()  # Ensure FIFO for equal priorities
+        heappush(self._heap, (priority, tiebreaker, item))
+        self._index[item] = len(self._heap) - 1
+
+    def extract(self, top_n = 1) -> P:
+        items = []
+        for _ in range(top_n):
+            if not self._heap:
+                break
+            priority, _, item = heappop(self._heap)
+            del self._index[item]
+            items.append(item)
+        if not items:
+            raise IndexError("Priority queue empty")
+        return items
+        # while self._heap:
+        #     _, _, item = heappop(self._heap)
+        #     if item in self._index:
+        #         del self._index[item]
+        #         return item
+        raise IndexError("Priority queue empty")
+
+
+    def is_empty(self) -> bool:
+        return not bool(self._heap)
+
+class BloomFilter:
+    """Optimal Bloom filter using murmur3 hash avalanche"""
+    __slots__ = ('size', 'hashes', 'bits')
+
+    def __init__(self, capacity: int, error_rate: float):
+        self.size = self._optimal_size(capacity, error_rate)
+        self.hashes = self._optimal_hashes(capacity, self.size)
+        self.bits = bitarray(self.size)
+        self.bits.setall(False)
+
+    @staticmethod
+    def _optimal_size(n: int, p: float) -> int:
+        m = - (n * np.log(p)) / (np.log(2) ** 2)
+        return int(np.ceil(m))
+
+    @staticmethod
+    def _optimal_hashes(n: int, m: int) -> int:
+        k = (m / n) * np.log(2)
+        return int(np.ceil(k))
+
+    def add(self, item: str) -> None:
+        for seed in range(self.hashes):
+            digest = mmh3.hash(item, seed) % self.size
+            self.bits[digest] = True
+
+    def __contains__(self, item: str) -> bool:
+        return all(
+            self.bits[mmh3.hash(item, seed) % self.size]
+            for seed in range(self.hashes)
+        )
+
+    def copy(self) -> BloomFilter:
+        new = object.__new__(BloomFilter)
+        new.size = self.size
+        new.hashes = self.hashes
+        new.bits = self.bits.copy()
+        return new
+    
+    def __len__(self) -> int:
+        """
+        Estimates the number of items in the filter using the 
+        count of set bits and the formula:
+        n = -m/k * ln(1 - X/m)
+        where:
+            m = size of bit array
+            k = number of hash functions
+            X = count of set bits
+        """
+        set_bits = self.bits.count(True)
+        if set_bits == 0:
+            return 0
+            
+        # Use the inverse bloom filter formula to estimate cardinality
+        return int(
+            -(self.size / self.hashes) * 
+            np.log(1 - set_bits / self.size)
+        )
+    
+    def bit_count(self) -> int:
+        """Returns the raw count of set bits in the filter"""
+        return self.bits.count(True)
+        
+    def __repr__(self) -> str:
+        return f"BloomFilter(est_items={len(self)}, bits={self.bit_count()}/{self.size})"
+
+# ------ Hyper-Optimal Deep Crawl Core ------ #
+class DeepCrawlDecorator:
+    """Metaprogramming marvel: Zero-cost deep crawl abstraction"""
+    def __init__(self, crawler: AsyncWebCrawler):
+        self.crawler = crawler
+
+    def __call__(self, original_arun: Callable) -> Callable:
+        @wraps(original_arun)
+        async def quantum_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
+            stack = deep_crawl_ctx.get()
+            if config and config.deep_crawl_strategy and not stack:
+                stack.append(self.crawler)
+                try:
+                    deep_crawl_ctx.set(stack)
+                    async for result in config.deep_crawl_strategy.traverse(
+                        start_url=url,
+                        crawler=self.crawler,
+                        config=config
+                    ):
+                        yield result
+                finally:
+                    stack.pop()
+                    deep_crawl_ctx.set(stack)
+            else:
+                result = await original_arun(url, config=config, **kwargs)
+                yield result
+        return quantum_arun
+
+
+async def collect_results(url, crawler, config):
+    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
+        setattr(crawler, "arun", getattr(crawler, "original_arun"))
+
+    ret = crawler.arun(url, config=config)
+    # If arun is an async generator, iterate over it
+    if inspect.isasyncgen(ret):
+        return [r async for r in ret]
+    # Otherwise, await the coroutine and normalize to a list
+    result = await ret
+    return result if isinstance(result, list) else [result]
+
+async def collect_many_results(url, crawler, config):
+    # Replace back arun to its original implementation
+    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
+        setattr(crawler, "arun", getattr(crawler, "original_arun"))
+    ret = crawler.arun_many(url, config=config)
+    # If arun is an async generator, iterate over it
+    if inspect.isasyncgen(ret):
+        return [r async for r in ret]
+    # Otherwise, await the coroutine and normalize to a list
+    result = await ret
+    return result if isinstance(result, list) else [result]
+
+
+# ------ Deep Crawl Strategy Interface ------ #
+CrawlResultT = TypeVar("CrawlResultT", bound=CrawlResult)
+# In batch mode we return List[CrawlResult] and in stream mode an AsyncGenerator.
+RunManyReturn = Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+
+
+class DeepCrawlStrategy(ABC):
+    """Abstract base class that will make Dijkstra smile"""
+    @abstractmethod
+    async def traverse(self,
+                      start_url: str,
+                      crawler: AsyncWebCrawler,
+                      config: CrawlerRunConfig) -> RunManyReturn:
+        """Traverse with O(1) memory complexity via generator fusion"""
+        ...
+
+    @abstractmethod
+    def precompute_priority(self, url: str) -> Awaitable[float]:
+        """Quantum-inspired priority precomputation"""
+        pass
+
+    @abstractmethod
+    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
+        """Hilbert-curve optimized link generation"""
+        pass
+
+# ------ BFS That Would Make Knuth Proud ------ #
+
+def calculate_quantum_batch_size(
+    depth: int,
+    max_depth: int,
+    frontier_size: int,
+    visited_size: int
+) -> int:
+    """
+    Calculates optimal batch size for URL processing using quantum-inspired mathematical principles.
+    
+    This function implements a sophisticated batch size calculation using:
+    1. Golden Ratio (φ) based scaling for optimal irrationality
+    2. Depth-aware amplitude modulation
+    3. Harmonic series dampening
+    4. Logarithmic growth control
+    5. Dynamic frontier adaptation
+    
+    The formula follows the quantum harmonic oscillator principle:
+        N = ⌈φ^(2d) * log₂(|V|) * H(d)⁻¹ * min(20, |F|/10)⌉
+    where:
+        φ = Golden Ratio ((1 + √5) / 2)
+        d = depth factor (normalized remaining depth)
+        |V| = size of visited set
+        H(d) = d-th harmonic number
+        |F| = frontier size
+    
+    Args:
+        depth (int): Current traversal depth
+        max_depth (int): Maximum allowed depth
+        frontier_size (int): Current size of frontier queue
+        visited_size (int): Number of URLs visited so far
+    
+    Returns:
+        int: Optimal batch size bounded between 1 and 100
+        
+    Mathematical Properties:
+        - Maintains O(log n) growth with respect to visited size
+        - Provides φ-optimal distribution of resources
+        - Ensures quantum-like state transitions between depths
+        - Harmonically dampened to prevent exponential explosion
+    """
+    # Golden ratio φ = (1 + √5) / 2
+    φ = (1 + 5 ** 0.5) / 2
+    
+    # Calculate normalized depth factor [0, 1]
+    depth_factor = (max_depth - depth) / max_depth if depth < max_depth else 0
+    
+    # Compute harmonic number for current depth
+    harmonic = sum(1/k for k in range(1, depth + 2))
+    
+    # Calculate quantum batch size
+    batch_size = int(np.ceil(
+        (φ ** (depth_factor * 2)) *          # Golden ratio scaling
+        np.log2(visited_size + 2) *          # Logarithmic growth factor
+        (1 / harmonic) *                     # Harmonic dampening
+        max(1, min(20, frontier_size / 10))  # Frontier-aware scaling
+    ))
+    
+    # Enforce practical bounds
+    return max(1, min(100, batch_size))
+
+
+class BFSDeepCrawlStrategy(DeepCrawlStrategy):
+    """Breadth-First Search with Einstein-Rosen bridge optimization"""
+    __slots__ = ('max_depth', 'filter_chain', 'priority_fn', 'stats', '_cancel')
+
+    def __init__(self,
+                 max_depth: int,
+                 filter_chain: FilterChain = FilterChain(),
+                 priority_fn: Callable[[str], Awaitable[float]] = lambda url: 1.0,
+                 logger: logging.Logger = None):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.priority_fn = priority_fn
+        self.stats = TraversalStats()
+        self._cancel = asyncio.Event()
+        self.semaphore = asyncio.Semaphore(1000)
+
+    async def traverse(self,
+                      start_url: str,
+                      crawler: AsyncWebCrawler,
+                      config: CrawlerRunConfig) -> RunManyReturn:
+        """Non-blocking BFS with O(b^d) time complexity awareness"""
+        ctx = TraversalContext(self.priority_fn)
+        ctx.frontier.insert(self.priority_fn(start_url), (start_url, None, 0))
+        ctx.visited.add(start_url)
+        ctx.depths[start_url] = 0
+
+        while not ctx.frontier.is_empty() and not self._cancel.is_set():
+            # Use the best algorith, to find top_n value
+            top_n = calculate_quantum_batch_size(
+                depth=ctx.current_depth,
+                max_depth=self.max_depth,
+                frontier_size=len(ctx.frontier._heap),
+                visited_size=len(ctx.visited)
+            )
+
+            urls = ctx.frontier.extract(top_n=top_n)
+            # url, parent, depth = ctx.frontier.extract(top_n=top_n)
+            if urls:
+                ctx.current_depth = urls[0][2]
+
+            async with self.semaphore:
+                results = await collect_many_results([url for (url, parent, depth) in urls], crawler, config)
+                # results = await asyncio.gather(*[
+                #     collect_results(url, crawler, config) for (url, parent, depth) in urls
+                # ])
+                # result = _result[0]
+                for ix, result in enumerate(results):
+                    url, parent, depth = result.url, urls[ix][1], urls[ix][2]
+                    result.metadata['depth'] = depth
+                    result.metadata['parent'] = parent
+                    yield result
+
+                    if depth < self.max_depth:
+                        async for link in self.link_hypercube(result):
+                            if link not in ctx.visited:
+                                priority = self.priority_fn(link)
+                                ctx.frontier.insert(priority, (link, url, depth + 1))
+                                ctx.visited.add(link)
+                                ctx.depths[link] = depth + 1
+
+    @lru_cache(maxsize=65536)
+    async def validate_url(self, url: str) -> bool:
+        """Memoized URL validation with λ-calculus purity"""
+        try:
+            parsed = urlparse(url)
+            return (parsed.scheme in {'http', 'https'}
+                    and '.' in parsed.netloc
+                    and await self.filter_chain.apply(url))
+        except Exception:
+            return False
+
+    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
+        """Hilbert-ordered link generation with O(1) yield latency"""
+        links = (link['href'] for link in result.links.get('internal', []))
+        validated = filter(self.validate_url, links)
+        for link in sorted(validated, key=lambda x: -self.priority_fn(x)):
+            yield link
+
+    def __aiter__(self) -> AsyncGenerator[CrawlResult, None]:
+        """Native async iterator interface"""
+        return self.traverse()
+
+    async def __anext__(self) -> CrawlResult:
+        """True async iterator protocol implementation"""
+        result = await self.traverse().__anext__()
+        if result:
+            return result
+        raise StopAsyncIteration
+
+    async def precompute_priority(self, url):
+        return super().precompute_priority(url)
+
+    async def shutdown(self):
+        self._cancel.set()
+
+# ------ Usage That Will Drop Jaws ------ #
+async def main():
+    """Quantum crawl example"""
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=2,
+        priority_fn=lambda url: 1.0 / (len(url) + 1e-9),  # Inverse length priority
+        # filter_chain=FilterChain(...)
+    )
+
+    config: CrawlerRunConfig = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        stream=False,
+        verbose=True,
+        cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        run_decorator = DeepCrawlDecorator(crawler)
+        setattr(crawler, "original_arun", crawler.arun)
+        crawler.arun = run_decorator(crawler.arun)
+        start_time = time.perf_counter()
+        async for result in crawler.arun("https://docs.crawl4ai.com", config=config):
+            print(f"🌀 {result.url} (Depth: {result.metadata['depth']})")
+        print(f"Deep crawl completed in {time.perf_counter() - start_time:.2f}s")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

crawl4ai/deep_crawling/dfs_strategy.py

@@ -0,0 +1,102 @@
+# dfs_deep_crawl_strategy.py
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+
+from ..models import CrawlResult
+from .bfs_strategy import BFSDeepCrawlStrategy  # noqa
+from ..types import AsyncWebCrawler, CrawlerRunConfig
+
+class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
+    """
+    Depth-First Search (DFS) deep crawling strategy.
+
+    Inherits URL validation and link discovery from BFSDeepCrawlStrategy.
+    Overrides _arun_batch and _arun_stream to use a stack (LIFO) for DFS traversal.
+    """
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) DFS mode.
+        Uses a stack to traverse URLs in DFS order, aggregating CrawlResults into a list.
+        """
+        visited: Set[str] = set()
+        # Stack items: (url, parent_url, depth)
+        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+        depths: Dict[str, int] = {start_url: 0}
+        results: List[CrawlResult] = []
+
+        while stack and not self._cancel_event.is_set():
+            url, parent, depth = stack.pop()
+            if url in visited or depth > self.max_depth:
+                continue
+            visited.add(url)
+
+            # Clone config to disable recursive deep crawling.
+            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
+            url_results = await crawler.arun_many(urls=[url], config=batch_config)
+            
+            for result in url_results:
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent
+                if self.url_scorer:
+                    result.metadata["score"] = self.url_scorer.score(url)
+                results.append(result)
+                
+                # Count only successful crawls toward max_pages limit
+                if result.success:
+                    self._pages_crawled += 1
+                    
+                    # Only discover links from successful crawls
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    
+                    # Push new links in reverse order so the first discovered is processed next.
+                    for new_url, new_parent in reversed(new_links):
+                        new_depth = depths.get(new_url, depth + 1)
+                        stack.append((new_url, new_parent, new_depth))
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming DFS mode.
+        Uses a stack to traverse URLs in DFS order and yields CrawlResults as they become available.
+        """
+        visited: Set[str] = set()
+        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        while stack and not self._cancel_event.is_set():
+            url, parent, depth = stack.pop()
+            if url in visited or depth > self.max_depth:
+                continue
+            visited.add(url)
+
+            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=[url], config=stream_config)
+            async for result in stream_gen:
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent
+                if self.url_scorer:
+                    result.metadata["score"] = self.url_scorer.score(url)
+                yield result
+
+                # Only count successful crawls toward max_pages limit
+                # and only discover links from successful crawls
+                if result.success:
+                    self._pages_crawled += 1
+                    
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    for new_url, new_parent in reversed(new_links):
+                        new_depth = depths.get(new_url, depth + 1)
+                        stack.append((new_url, new_parent, new_depth))

crawl4ai/deep_crawling/filters.py

@@ -0,0 +1,666 @@
+from abc import ABC, abstractmethod
+from typing import List, Pattern, Set, Union
+from urllib.parse import urlparse
+from array import array
+import re
+import logging
+from functools import lru_cache
+import fnmatch
+from dataclasses import dataclass
+import weakref
+import math
+from collections import defaultdict
+from typing import Dict
+from ..utils import HeadPeekr
+import asyncio
+import inspect
+
+
+@dataclass
+class FilterStats:
+    __slots__ = ("_counters",)
+
+    def __init__(self):
+        # Use array of unsigned ints for atomic operations
+        self._counters = array("I", [0, 0, 0])  # total, passed, rejected
+
+    @property
+    def total_urls(self):
+        return self._counters[0]
+
+    @property
+    def passed_urls(self):
+        return self._counters[1]
+
+    @property
+    def rejected_urls(self):
+        return self._counters[2]
+
+
+class URLFilter(ABC):
+    """Optimized base filter class"""
+
+    __slots__ = ("name", "stats", "_logger_ref")
+
+    def __init__(self, name: str = None):
+        self.name = name or self.__class__.__name__
+        self.stats = FilterStats()
+        # Lazy logger initialization using weakref
+        self._logger_ref = None
+
+    @property
+    def logger(self):
+        if self._logger_ref is None or self._logger_ref() is None:
+            logger = logging.getLogger(f"urlfilter.{self.name}")
+            self._logger_ref = weakref.ref(logger)
+        return self._logger_ref()
+
+    @abstractmethod
+    def apply(self, url: str) -> bool:
+        pass
+
+    def _update_stats(self, passed: bool):
+        # Use direct array index for speed
+        self.stats._counters[0] += 1  # total
+        self.stats._counters[1] += passed  # passed
+        self.stats._counters[2] += not passed  # rejected
+
+
+class FilterChain:
+    """Optimized filter chain"""
+
+    __slots__ = ("filters", "stats", "_logger_ref")
+
+    def __init__(self, filters: List[URLFilter] = None):
+        self.filters = tuple(filters or [])  # Immutable tuple for speed
+        self.stats = FilterStats()
+        self._logger_ref = None
+
+    @property
+    def logger(self):
+        if self._logger_ref is None or self._logger_ref() is None:
+            logger = logging.getLogger("urlfilter.chain")
+            self._logger_ref = weakref.ref(logger)
+        return self._logger_ref()
+
+    def add_filter(self, filter_: URLFilter) -> "FilterChain":
+        """Add a filter to the chain"""
+        self.filters.append(filter_)
+        return self  # Enable method chaining
+
+    async def apply(self, url: str) -> bool:
+        """Apply all filters concurrently when possible"""
+        self.stats._counters[0] += 1  # Total processed URLs
+
+        tasks = []
+        for f in self.filters:
+            result = f.apply(url)
+
+            if inspect.isawaitable(result):
+                tasks.append(result)  # Collect async tasks
+            elif not result:  # Sync rejection
+                self.stats._counters[2] += 1  # Sync rejected
+                return False
+
+        if tasks:
+            results = await asyncio.gather(*tasks)
+
+            # Count how many filters rejected
+            rejections = results.count(False)
+            self.stats._counters[2] += rejections
+
+            if not all(results):
+                return False  # Stop early if any filter rejected
+
+        self.stats._counters[1] += 1  # Passed
+        return True
+
+
+class URLPatternFilter(URLFilter):
+    """Pattern filter balancing speed and completeness"""
+
+    __slots__ = (
+        "_simple_suffixes",
+        "_simple_prefixes",
+        "_domain_patterns",
+        "_path_patterns",
+        "_reverse",
+    )
+
+    PATTERN_TYPES = {
+        "SUFFIX": 1,  # *.html
+        "PREFIX": 2,  # /foo/*
+        "DOMAIN": 3,  # *.example.com
+        "PATH": 4,  # Everything else
+        "REGEX": 5,
+    }
+
+    def __init__(
+        self,
+        patterns: Union[str, Pattern, List[Union[str, Pattern]]],
+        use_glob: bool = True,
+        reverse: bool = False,
+    ):
+        super().__init__()
+        self._reverse = reverse
+        patterns = [patterns] if isinstance(patterns, (str, Pattern)) else patterns
+
+        self._simple_suffixes = set()
+        self._simple_prefixes = set()
+        self._domain_patterns = []
+        self._path_patterns = []
+
+        for pattern in patterns:
+            pattern_type = self._categorize_pattern(pattern)
+            self._add_pattern(pattern, pattern_type)
+
+    def _categorize_pattern(self, pattern: str) -> int:
+        """Categorize pattern for specialized handling"""
+        if not isinstance(pattern, str):
+            return self.PATTERN_TYPES["PATH"]
+
+        # Check if it's a regex pattern
+        if pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern:
+            return self.PATTERN_TYPES["REGEX"]
+
+        if pattern.count("*") == 1:
+            if pattern.startswith("*."):
+                return self.PATTERN_TYPES["SUFFIX"]
+            if pattern.endswith("/*"):
+                return self.PATTERN_TYPES["PREFIX"]
+
+        if "://" in pattern and pattern.startswith("*."):
+            return self.PATTERN_TYPES["DOMAIN"]
+
+        return self.PATTERN_TYPES["PATH"]
+
+    def _add_pattern(self, pattern: str, pattern_type: int):
+        """Add pattern to appropriate matcher"""
+        if pattern_type == self.PATTERN_TYPES["REGEX"]:
+            # For regex patterns, compile directly without glob translation
+            if isinstance(pattern, str) and (
+                pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern
+            ):
+                self._path_patterns.append(re.compile(pattern))
+                return
+        elif pattern_type == self.PATTERN_TYPES["SUFFIX"]:
+            self._simple_suffixes.add(pattern[2:])
+        elif pattern_type == self.PATTERN_TYPES["PREFIX"]:
+            self._simple_prefixes.add(pattern[:-2])
+        elif pattern_type == self.PATTERN_TYPES["DOMAIN"]:
+            self._domain_patterns.append(re.compile(pattern.replace("*.", r"[^/]+\.")))
+        else:
+            if isinstance(pattern, str):
+                # Handle complex glob patterns
+                if "**" in pattern:
+                    pattern = pattern.replace("**", ".*")
+                if "{" in pattern:
+                    # Convert {a,b} to (a|b)
+                    pattern = re.sub(
+                        r"\{([^}]+)\}",
+                        lambda m: f'({"|".join(m.group(1).split(","))})',
+                        pattern,
+                    )
+                pattern = fnmatch.translate(pattern)
+            self._path_patterns.append(
+                pattern if isinstance(pattern, Pattern) else re.compile(pattern)
+            )
+
+    @lru_cache(maxsize=10000)
+    def apply(self, url: str) -> bool:
+        # Quick suffix check (*.html)
+        if self._simple_suffixes:
+            path = url.split("?")[0]
+            if path.split("/")[-1].split(".")[-1] in self._simple_suffixes:
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
+
+        # Domain check
+        if self._domain_patterns:
+            for pattern in self._domain_patterns:
+                if pattern.match(url):
+                    result = True
+                    self._update_stats(result)
+                    return not result if self._reverse else result
+
+        # Prefix check (/foo/*)
+        if self._simple_prefixes:
+            path = url.split("?")[0]
+            if any(path.startswith(p) for p in self._simple_prefixes):
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
+
+        # Complex patterns
+        if self._path_patterns:
+            if any(p.search(url) for p in self._path_patterns):
+                result = True
+                self._update_stats(result)
+                return not result if self._reverse else result
+
+        result = False
+        self._update_stats(result)
+        return not result if self._reverse else result
+
+
+class ContentTypeFilter(URLFilter):
+    """Optimized content type filter using fast lookups"""
+
+    __slots__ = ("allowed_types", "_ext_map", "_check_extension")
+
+    # Fast extension to mime type mapping
+    _MIME_MAP = {
+        # Text Formats
+        "txt": "text/plain",
+        "html": "text/html",
+        "htm": "text/html",
+        "xhtml": "application/xhtml+xml",
+        "css": "text/css",
+        "csv": "text/csv",
+        "ics": "text/calendar",
+        "js": "application/javascript",
+        # Images
+        "bmp": "image/bmp",
+        "gif": "image/gif",
+        "jpeg": "image/jpeg",
+        "jpg": "image/jpeg",
+        "png": "image/png",
+        "svg": "image/svg+xml",
+        "tiff": "image/tiff",
+        "ico": "image/x-icon",
+        "webp": "image/webp",
+        # Audio
+        "mp3": "audio/mpeg",
+        "wav": "audio/wav",
+        "ogg": "audio/ogg",
+        "m4a": "audio/mp4",
+        "aac": "audio/aac",
+        # Video
+        "mp4": "video/mp4",
+        "mpeg": "video/mpeg",
+        "webm": "video/webm",
+        "avi": "video/x-msvideo",
+        "mov": "video/quicktime",
+        "flv": "video/x-flv",
+        "wmv": "video/x-ms-wmv",
+        "mkv": "video/x-matroska",
+        # Applications
+        "json": "application/json",
+        "xml": "application/xml",
+        "pdf": "application/pdf",
+        "zip": "application/zip",
+        "gz": "application/gzip",
+        "tar": "application/x-tar",
+        "rar": "application/vnd.rar",
+        "7z": "application/x-7z-compressed",
+        "exe": "application/vnd.microsoft.portable-executable",
+        "msi": "application/x-msdownload",
+        # Fonts
+        "woff": "font/woff",
+        "woff2": "font/woff2",
+        "ttf": "font/ttf",
+        "otf": "font/otf",
+        # Microsoft Office
+        "doc": "application/msword",
+        "dot": "application/msword",
+        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "xls": "application/vnd.ms-excel",
+        "ppt": "application/vnd.ms-powerpoint",
+        "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        # OpenDocument Formats
+        "odt": "application/vnd.oasis.opendocument.text",
+        "ods": "application/vnd.oasis.opendocument.spreadsheet",
+        "odp": "application/vnd.oasis.opendocument.presentation",
+        # Archives
+        "tar.gz": "application/gzip",
+        "tgz": "application/gzip",
+        "bz2": "application/x-bzip2",
+        # Others
+        "rtf": "application/rtf",
+        "apk": "application/vnd.android.package-archive",
+        "epub": "application/epub+zip",
+        "jar": "application/java-archive",
+        "swf": "application/x-shockwave-flash",
+        "midi": "audio/midi",
+        "mid": "audio/midi",
+        "ps": "application/postscript",
+        "ai": "application/postscript",
+        "eps": "application/postscript",
+        # Custom or less common
+        "bin": "application/octet-stream",
+        "dmg": "application/x-apple-diskimage",
+        "iso": "application/x-iso9660-image",
+        "deb": "application/x-debian-package",
+        "rpm": "application/x-rpm",
+        "sqlite": "application/vnd.sqlite3",
+        # Placeholder
+        "unknown": "application/octet-stream",  # Fallback for unknown file types
+    }
+
+    @staticmethod
+    @lru_cache(maxsize=1000)
+    def _extract_extension(url: str) -> str:
+        """Extracts file extension from a URL."""
+        # Remove scheme (http://, https://) if present
+        if "://" in url:
+            url = url.split("://", 1)[-1]  # Get everything after '://'
+
+        # Remove domain (everything up to the first '/')
+        path_start = url.find("/")
+        path = url[path_start:] if path_start != -1 else ""
+
+        # Extract last filename in path
+        filename = path.rsplit("/", 1)[-1] if "/" in path else ""
+
+        # Extract and validate extension
+        if "." not in filename:
+            return ""
+
+        return filename.rpartition(".")[-1].lower()
+
+    def __init__(
+        self,
+        allowed_types: Union[str, List[str]],
+        check_extension: bool = True,
+        ext_map: Dict[str, str] = _MIME_MAP,
+    ):
+        super().__init__()
+        # Normalize and store as frozenset for fast lookup
+        self.allowed_types = frozenset(
+            t.lower()
+            for t in (
+                allowed_types if isinstance(allowed_types, list) else [allowed_types]
+            )
+        )
+        self._check_extension = check_extension
+
+        # Pre-compute extension map for allowed types
+        self._ext_map = frozenset(
+            ext
+            for ext, mime in self._MIME_MAP.items()
+            if any(allowed in mime for allowed in self.allowed_types)
+        )
+
+    @lru_cache(maxsize=1000)
+    def _check_url_cached(self, url: str) -> bool:
+        """Cached URL checking"""
+        if not self._check_extension:
+            return True
+        ext = self._extract_extension(url)
+        if not ext:
+            return True
+
+        return ext in self._ext_map
+
+    def apply(self, url: str) -> bool:
+        """Fast extension check with caching"""
+        result = self._check_url_cached(url)
+        self._update_stats(result)
+        return result
+
+
+class DomainFilter(URLFilter):
+    """Optimized domain filter with fast lookups and caching"""
+
+    __slots__ = ("_allowed_domains", "_blocked_domains", "_domain_cache")
+
+    # Regex for fast domain extraction
+    _DOMAIN_REGEX = re.compile(r"://([^/]+)")
+
+    def __init__(
+        self,
+        allowed_domains: Union[str, List[str]] = None,
+        blocked_domains: Union[str, List[str]] = None,
+    ):
+        super().__init__()
+
+        # Convert inputs to frozensets for immutable, fast lookups
+        self._allowed_domains = (
+            frozenset(self._normalize_domains(allowed_domains))
+            if allowed_domains
+            else None
+        )
+        self._blocked_domains = (
+            frozenset(self._normalize_domains(blocked_domains))
+            if blocked_domains
+            else frozenset()
+        )
+
+    @staticmethod
+    def _normalize_domains(domains: Union[str, List[str]]) -> Set[str]:
+        """Fast domain normalization"""
+        if isinstance(domains, str):
+            return {domains.lower()}
+        return {d.lower() for d in domains}
+    
+    @staticmethod
+    def _is_subdomain(domain: str, parent_domain: str) -> bool:
+        """Check if domain is a subdomain of parent_domain"""
+        return domain == parent_domain or domain.endswith(f".{parent_domain}")
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _extract_domain(url: str) -> str:
+        """Ultra-fast domain extraction with regex and caching"""
+        match = DomainFilter._DOMAIN_REGEX.search(url)
+        return match.group(1).lower() if match else ""
+
+    def apply(self, url: str) -> bool:
+        """Optimized domain checking with early returns"""
+        # Skip processing if no filters
+        if not self._blocked_domains and self._allowed_domains is None:
+            self._update_stats(True)
+            return True
+
+        domain = self._extract_domain(url)
+
+        # Check for blocked domains, including subdomains
+        for blocked in self._blocked_domains:
+            if self._is_subdomain(domain, blocked):
+                self._update_stats(False)
+                return False
+
+        # If no allowed domains specified, accept all non-blocked
+        if self._allowed_domains is None:
+            self._update_stats(True)
+            return True
+
+        # Check if domain matches any allowed domain (including subdomains)
+        for allowed in self._allowed_domains:
+            if self._is_subdomain(domain, allowed):
+                self._update_stats(True)
+                return True
+
+        # No matches found
+        self._update_stats(False)
+        return False
+
+
+class ContentRelevanceFilter(URLFilter):
+    """BM25-based relevance filter using head section content"""
+
+    __slots__ = ("query_terms", "threshold", "k1", "b", "avgdl")
+
+    def __init__(
+        self,
+        query: str,
+        threshold: float,
+        k1: float = 1.2,
+        b: float = 0.75,
+        avgdl: int = 1000,
+    ):
+        super().__init__(name="BM25RelevanceFilter")
+        self.query_terms = self._tokenize(query)
+        self.threshold = threshold
+        self.k1 = k1  # TF saturation parameter
+        self.b = b  # Length normalization parameter
+        self.avgdl = avgdl  # Average document length (empirical value)
+
+    async def apply(self, url: str) -> bool:
+        head_content = await HeadPeekr.peek_html(url)
+        if not head_content:
+            self._update_stats(False)
+            return False
+
+        # Field extraction with weighting
+        fields = {
+            "title": HeadPeekr.get_title(head_content) or "",
+            "meta": HeadPeekr.extract_meta_tags(head_content),
+        }
+        doc_text = self._build_document(fields)
+
+        score = self._bm25(doc_text)
+        decision = score >= self.threshold
+        self._update_stats(decision)
+        return decision
+
+    def _build_document(self, fields: Dict) -> str:
+        """Weighted document construction"""
+        return " ".join(
+            [
+                fields["title"] * 3,  # Title weight
+                fields["meta"].get("description", "") * 2,
+                fields["meta"].get("keywords", ""),
+                " ".join(fields["meta"].values()),
+            ]
+        )
+
+    def _tokenize(self, text: str) -> List[str]:
+        """Fast case-insensitive tokenization"""
+        return text.lower().split()
+
+    def _bm25(self, document: str) -> float:
+        """Optimized BM25 implementation for head sections"""
+        doc_terms = self._tokenize(document)
+        doc_len = len(doc_terms)
+        tf = defaultdict(int)
+
+        for term in doc_terms:
+            tf[term] += 1
+
+        score = 0.0
+        for term in set(self.query_terms):
+            term_freq = tf[term]
+            idf = math.log((1 + 1) / (term_freq + 0.5) + 1)  # Simplified IDF
+            numerator = term_freq * (self.k1 + 1)
+            denominator = term_freq + self.k1 * (
+                1 - self.b + self.b * (doc_len / self.avgdl)
+            )
+            score += idf * (numerator / denominator)
+
+        return score
+
+
+class SEOFilter(URLFilter):
+    """Quantitative SEO quality assessment filter using head section analysis"""
+
+    __slots__ = ("threshold", "_weights", "_kw_patterns")
+
+    # Based on SEMrush/Google ranking factors research
+    DEFAULT_WEIGHTS = {
+        "title_length": 0.15,
+        "title_kw": 0.18,
+        "meta_description": 0.12,
+        "canonical": 0.10,
+        "robot_ok": 0.20,  # Most critical factor
+        "schema_org": 0.10,
+        "url_quality": 0.15,
+    }
+
+    def __init__(
+        self,
+        threshold: float = 0.65,
+        keywords: List[str] = None,
+        weights: Dict[str, float] = None,
+    ):
+        super().__init__(name="SEOFilter")
+        self.threshold = threshold
+        self._weights = weights or self.DEFAULT_WEIGHTS
+        self._kw_patterns = (
+            re.compile(
+                r"\b({})\b".format("|".join(map(re.escape, keywords or []))), re.I
+            )
+            if keywords
+            else None
+        )
+
+    async def apply(self, url: str) -> bool:
+        head_content = await HeadPeekr.peek_html(url)
+        if not head_content:
+            self._update_stats(False)
+            return False
+
+        meta = HeadPeekr.extract_meta_tags(head_content)
+        title = HeadPeekr.get_title(head_content) or ""
+        parsed_url = urlparse(url)
+
+        scores = {
+            "title_length": self._score_title_length(title),
+            "title_kw": self._score_keyword_presence(title),
+            "meta_description": self._score_meta_description(
+                meta.get("description", "")
+            ),
+            "canonical": self._score_canonical(meta.get("canonical"), url),
+            "robot_ok": 1.0 if "noindex" not in meta.get("robots", "") else 0.0,
+            "schema_org": self._score_schema_org(head_content),
+            "url_quality": self._score_url_quality(parsed_url),
+        }
+
+        total_score = sum(
+            weight * scores[factor] for factor, weight in self._weights.items()
+        )
+
+        decision = total_score >= self.threshold
+        self._update_stats(decision)
+        return decision
+
+    def _score_title_length(self, title: str) -> float:
+        length = len(title)
+        if 50 <= length <= 60:
+            return 1.0
+        if 40 <= length < 50 or 60 < length <= 70:
+            return 0.7
+        return 0.3  # Poor length
+
+    def _score_keyword_presence(self, text: str) -> float:
+        if not self._kw_patterns:
+            return 0.0
+        matches = len(self._kw_patterns.findall(text))
+        return min(matches * 0.3, 1.0)  # Max 3 matches
+
+    def _score_meta_description(self, desc: str) -> float:
+        length = len(desc)
+        if 140 <= length <= 160:
+            return 1.0
+        return 0.5 if 120 <= length <= 200 else 0.2
+
+    def _score_canonical(self, canonical: str, original: str) -> float:
+        if not canonical:
+            return 0.5  # Neutral score
+        return 1.0 if canonical == original else 0.2
+
+    def _score_schema_org(self, html: str) -> float:
+        # Detect any schema.org markup in head
+        return (
+            1.0
+            if re.search(r'<script[^>]+type=["\']application/ld\+json', html)
+            else 0.0
+        )
+
+    def _score_url_quality(self, parsed_url) -> float:
+        score = 1.0
+        path = parsed_url.path.lower()
+
+        # Penalty factors
+        if len(path) > 80:
+            score *= 0.7
+        if re.search(r"\d{4}", path):
+            score *= 0.8  # Numbers in path
+        if parsed_url.query:
+            score *= 0.6  # URL parameters
+        if "_" in path:
+            score *= 0.9  # Underscores vs hyphens
+
+        return score

crawl4ai/deep_crawling/scorers.py

@@ -0,0 +1,519 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from urllib.parse import urlparse, unquote
+import re
+import logging
+from functools import lru_cache
+from array import array
+import ctypes
+import platform
+PLATFORM = platform.system()
+
+# Pre-computed scores for common year differences
+_SCORE_LOOKUP = [1.0, 0.5, 0.3333333333333333, 0.25]
+
+# Pre-computed scores for common year differences
+_FRESHNESS_SCORES = [
+   1.0,    # Current year
+   0.9,    # Last year
+   0.8,    # 2 years ago
+   0.7,    # 3 years ago
+   0.6,    # 4 years ago
+   0.5,    # 5 years ago
+]
+
+class ScoringStats:
+    __slots__ = ('_urls_scored', '_total_score', '_min_score', '_max_score')
+    
+    def __init__(self):
+        self._urls_scored = 0
+        self._total_score = 0.0
+        self._min_score = None  # Lazy initialization
+        self._max_score = None
+    
+    def update(self, score: float) -> None:
+        """Optimized update with minimal operations"""
+        self._urls_scored += 1
+        self._total_score += score
+        
+        # Lazy min/max tracking - only if actually accessed
+        if self._min_score is not None:
+            if score < self._min_score:
+                self._min_score = score
+        if self._max_score is not None:
+            if score > self._max_score:
+                self._max_score = score
+                
+    def get_average(self) -> float:
+        """Direct calculation instead of property"""
+        return self._total_score / self._urls_scored if self._urls_scored else 0.0
+    
+    def get_min(self) -> float:
+        """Lazy min calculation"""
+        if self._min_score is None:
+            self._min_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
+        return self._min_score
+        
+    def get_max(self) -> float:
+        """Lazy max calculation"""
+        if self._max_score is None:
+            self._max_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
+        return self._max_score
+class URLScorer(ABC):
+    __slots__ = ('_weight', '_stats')
+    
+    def __init__(self, weight: float = 1.0):
+        # Store weight directly as float32 for memory efficiency
+        self._weight = ctypes.c_float(weight).value
+        self._stats = ScoringStats()
+    
+    @abstractmethod
+    def _calculate_score(self, url: str) -> float:
+        """Calculate raw score for URL."""
+        pass
+    
+    def score(self, url: str) -> float:
+        """Calculate weighted score with minimal overhead."""
+        score = self._calculate_score(url) * self._weight
+        self._stats.update(score)
+        return score
+    
+    @property
+    def stats(self):
+        """Access to scoring statistics."""
+        return self._stats
+    
+    @property
+    def weight(self):
+        return self._weight
+
+class CompositeScorer(URLScorer):
+    __slots__ = ('_scorers', '_normalize', '_weights_array', '_score_array')
+    
+    def __init__(self, scorers: List[URLScorer], normalize: bool = True):
+        """Initialize composite scorer combining multiple scoring strategies.
+        
+        Optimized for:
+        - Fast parallel scoring
+        - Memory efficient score aggregation
+        - Quick short-circuit conditions
+        - Pre-allocated arrays
+        
+        Args:
+            scorers: List of scoring strategies to combine
+            normalize: Whether to normalize final score by scorer count
+        """
+        super().__init__(weight=1.0)
+        self._scorers = scorers
+        self._normalize = normalize
+        
+        # Pre-allocate arrays for scores and weights
+        self._weights_array = array('f', [s.weight for s in scorers])
+        self._score_array = array('f', [0.0] * len(scorers))
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate combined score from all scoring strategies.
+        
+        Uses:
+        1. Pre-allocated arrays for scores
+        2. Short-circuit on zero scores
+        3. Optimized normalization
+        4. Vectorized operations where possible
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Combined and optionally normalized score
+        """
+        total_score = 0.0
+        scores = self._score_array
+        
+        # Get scores from all scorers
+        for i, scorer in enumerate(self._scorers):
+            # Use public score() method which applies weight
+            scores[i] = scorer.score(url)
+            total_score += scores[i]
+            
+        # Normalize if requested
+        if self._normalize and self._scorers:
+            count = len(self._scorers)
+            return total_score / count
+            
+        return total_score
+
+    def score(self, url: str) -> float:
+        """Public scoring interface with stats tracking.
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Final combined score
+        """
+        score = self._calculate_score(url)
+        self.stats.update(score)
+        return score
+
+class KeywordRelevanceScorer(URLScorer):
+    __slots__ = ('_weight', '_stats', '_keywords', '_case_sensitive')
+    
+    def __init__(self, keywords: List[str], weight: float = 1.0, case_sensitive: bool = False):
+        super().__init__(weight=weight)
+        self._case_sensitive = case_sensitive
+        # Pre-process keywords once
+        self._keywords = [k if case_sensitive else k.lower() for k in keywords]
+    
+    @lru_cache(maxsize=10000)
+    def _url_bytes(self, url: str) -> bytes:
+        """Cache decoded URL bytes"""
+        return url.encode('utf-8') if self._case_sensitive else url.lower().encode('utf-8')
+    
+    
+    def _calculate_score(self, url: str) -> float:
+        """Fast string matching without regex or byte conversion"""
+        if not self._case_sensitive:
+            url = url.lower()
+            
+        matches = sum(1 for k in self._keywords if k in url)
+        
+        # Fast return paths
+        if not matches:
+            return 0.0
+        if matches == len(self._keywords):
+            return 1.0
+            
+        return matches / len(self._keywords)
+
+class PathDepthScorer(URLScorer):
+    __slots__ = ('_weight', '_stats', '_optimal_depth')  # Remove _url_cache
+    
+    def __init__(self, optimal_depth: int = 3, weight: float = 1.0):
+        super().__init__(weight=weight)
+        self._optimal_depth = optimal_depth
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _quick_depth(path: str) -> int:
+        """Ultra fast path depth calculation.
+        
+        Examples:
+            - "http://example.com" -> 0  # No path segments
+            - "http://example.com/" -> 0  # Empty path
+            - "http://example.com/a" -> 1
+            - "http://example.com/a/b" -> 2
+        """
+        if not path or path == '/':
+            return 0
+            
+        if '/' not in path:
+            return 0
+            
+        depth = 0
+        last_was_slash = True
+        
+        for c in path:
+            if c == '/':
+                if not last_was_slash:
+                    depth += 1
+                last_was_slash = True
+            else:
+                last_was_slash = False
+                
+        if not last_was_slash:
+            depth += 1
+            
+        return depth
+
+    @lru_cache(maxsize=10000)  # Cache the whole calculation
+    def _calculate_score(self, url: str) -> float:
+        pos = url.find('/', url.find('://') + 3)
+        if pos == -1:
+            depth = 0
+        else:
+            depth = self._quick_depth(url[pos:])
+            
+        # Use lookup table for common distances
+        distance = depth - self._optimal_depth
+        distance = distance if distance >= 0 else -distance  # Faster than abs()
+        
+        if distance < 4:
+            return _SCORE_LOOKUP[distance]
+            
+        return 1.0 / (1.0 + distance)                                             
+
+class ContentTypeScorer(URLScorer):
+    __slots__ = ('_weight', '_exact_types', '_regex_types')
+
+    def __init__(self, type_weights: Dict[str, float], weight: float = 1.0):
+        """Initialize scorer with type weights map.
+        
+        Args:
+            type_weights: Dict mapping file extensions/patterns to scores (e.g. {'.html$': 1.0})
+            weight: Overall weight multiplier for this scorer
+        """
+        super().__init__(weight=weight)
+        self._exact_types = {}  # Fast lookup for simple extensions
+        self._regex_types = []  # Fallback for complex patterns
+        
+        # Split into exact vs regex matchers for performance
+        for pattern, score in type_weights.items():
+            if pattern.startswith('.') and pattern.endswith('$'):
+                ext = pattern[1:-1]
+                self._exact_types[ext] = score
+            else:
+                self._regex_types.append((re.compile(pattern), score))
+                
+        # Sort complex patterns by score for early exit
+        self._regex_types.sort(key=lambda x: -x[1])
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _quick_extension(url: str) -> str:
+        """Extract file extension ultra-fast without regex/splits.
+        
+        Handles:
+        - Basic extensions: "example.html" -> "html"
+        - Query strings: "page.php?id=1" -> "php" 
+        - Fragments: "doc.pdf#page=1" -> "pdf"
+        - Path params: "file.jpg;width=100" -> "jpg"
+        
+        Args:
+            url: URL to extract extension from
+            
+        Returns:
+            Extension without dot, or empty string if none found
+        """
+        pos = url.rfind('.')
+        if pos == -1:
+            return ''
+        
+        # Find first non-alphanumeric char after extension
+        end = len(url)
+        for i in range(pos + 1, len(url)):
+            c = url[i]
+            # Stop at query string, fragment, path param or any non-alphanumeric
+            if c in '?#;' or not c.isalnum():
+                end = i
+                break
+                
+        return url[pos + 1:end].lower()
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate content type score for URL.
+        
+        Uses staged approach:
+        1. Try exact extension match (fast path)
+        2. Fall back to regex patterns if needed
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Score between 0.0 and 1.0 * weight
+        """
+        # Fast path: direct extension lookup
+        ext = self._quick_extension(url)
+        if ext:
+            score = self._exact_types.get(ext, None)
+            if score is not None:
+                return score
+                
+        # Slow path: regex patterns
+        for pattern, score in self._regex_types:
+            if pattern.search(url):
+                return score
+
+        return 0.0
+
+class FreshnessScorer(URLScorer):
+    __slots__ = ('_weight', '_date_pattern', '_current_year')
+
+    def __init__(self, weight: float = 1.0, current_year: int = 2024):
+        """Initialize freshness scorer.
+        
+        Extracts and scores dates from URLs using format:
+        - YYYY/MM/DD 
+        - YYYY-MM-DD
+        - YYYY_MM_DD
+        - YYYY (year only)
+        
+        Args:
+            weight: Score multiplier
+            current_year: Year to calculate freshness against (default 2024)
+        """
+        super().__init__(weight=weight)
+        self._current_year = current_year
+        
+        # Combined pattern for all date formats
+        # Uses non-capturing groups (?:) and alternation
+        self._date_pattern = re.compile(
+            r'(?:/'  # Path separator
+            r'|[-_])'  # or date separators
+            r'((?:19|20)\d{2})'  # Year group (1900-2099)
+            r'(?:'  # Optional month/day group
+            r'(?:/|[-_])'  # Date separator  
+            r'(?:\d{2})'  # Month
+            r'(?:'  # Optional day
+            r'(?:/|[-_])'  # Date separator
+            r'(?:\d{2})'  # Day
+            r')?'  # Day is optional
+            r')?'  # Month/day group is optional
+        )
+
+    @lru_cache(maxsize=10000)
+    def _extract_year(self, url: str) -> Optional[int]:
+        """Extract the most recent year from URL.
+        
+        Args:
+            url: URL to extract year from
+            
+        Returns:
+            Year as int or None if no valid year found
+        """
+        matches = self._date_pattern.finditer(url)
+        latest_year = None
+        
+        # Find most recent year
+        for match in matches:
+            year = int(match.group(1))
+            if (year <= self._current_year and  # Sanity check
+                (latest_year is None or year > latest_year)):
+                latest_year = year
+                
+        return latest_year
+
+    @lru_cache(maxsize=10000) 
+    def _calculate_score(self, url: str) -> float:
+        """Calculate freshness score based on URL date.
+        
+        More recent years score higher. Uses pre-computed scoring
+        table for common year differences.
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Score between 0.0 and 1.0 * weight
+        """
+        year = self._extract_year(url)
+        if year is None:
+            return 0.5  # Default score
+            
+        # Use lookup table for common year differences
+        year_diff = self._current_year - year
+        if year_diff < len(_FRESHNESS_SCORES):
+            return _FRESHNESS_SCORES[year_diff]
+            
+        # Fallback calculation for older content
+        return max(0.1, 1.0 - year_diff * 0.1)
+
+class DomainAuthorityScorer(URLScorer):
+    __slots__ = ('_weight', '_domain_weights', '_default_weight', '_top_domains')
+    
+    def __init__(
+        self,
+        domain_weights: Dict[str, float],
+        default_weight: float = 0.5,
+        weight: float = 1.0,
+    ):
+        """Initialize domain authority scorer.
+        
+        Args:
+            domain_weights: Dict mapping domains to authority scores
+            default_weight: Score for unknown domains
+            weight: Overall scorer weight multiplier
+            
+        Example:
+            {
+                'python.org': 1.0,
+                'github.com': 0.9,
+                'medium.com': 0.7
+            }
+        """
+        super().__init__(weight=weight)
+        
+        # Pre-process domains for faster lookup
+        self._domain_weights = {
+            domain.lower(): score 
+            for domain, score in domain_weights.items()
+        }
+        self._default_weight = default_weight
+        
+        # Cache top domains for fast path
+        self._top_domains = {
+            domain: score
+            for domain, score in sorted(
+                domain_weights.items(), 
+                key=lambda x: -x[1]
+            )[:5]  # Keep top 5 highest scoring domains
+        }
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _extract_domain(url: str) -> str:
+        """Extract domain from URL ultra-fast.
+        
+        Handles:
+        - Basic domains: "example.com"
+        - Subdomains: "sub.example.com" 
+        - Ports: "example.com:8080"
+        - IPv4: "192.168.1.1"
+        
+        Args:
+            url: Full URL to extract domain from
+            
+        Returns:
+            Lowercase domain without port
+        """
+        # Find domain start
+        start = url.find('://') 
+        if start == -1:
+            start = 0
+        else:
+            start += 3
+            
+        # Find domain end
+        end = url.find('/', start)
+        if end == -1:
+            end = url.find('?', start)
+            if end == -1:
+                end = url.find('#', start)
+                if end == -1:
+                    end = len(url)
+                    
+        # Extract domain and remove port
+        domain = url[start:end]
+        port_idx = domain.rfind(':')
+        if port_idx != -1:
+            domain = domain[:port_idx]
+            
+        return domain.lower()
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate domain authority score.
+        
+        Uses staged approach:
+        1. Check top domains (fastest)
+        2. Check full domain weights
+        3. Return default weight
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Authority score between 0.0 and 1.0 * weight
+        """
+        domain = self._extract_domain(url)
+        
+        # Fast path: check top domains first
+        score = self._top_domains.get(domain)
+        if score is not None:
+            return score
+            
+        # Regular path: check all domains
+        return self._domain_weights.get(domain, self._default_weight)

crawl4ai/docker_client.py

@@ -0,0 +1,170 @@
+from typing import List, Optional, Union, AsyncGenerator, Dict, Any
+import httpx
+import json
+from urllib.parse import urljoin
+import asyncio
+
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from .models import CrawlResult
+from .async_logger import AsyncLogger, LogLevel
+
+
+class Crawl4aiClientError(Exception):
+    """Base exception for Crawl4ai Docker client errors."""
+    pass
+
+
+class ConnectionError(Crawl4aiClientError):
+    """Raised when connection to the Docker server fails."""
+    pass
+
+
+class RequestError(Crawl4aiClientError):
+    """Raised when the server returns an error response."""
+    pass
+
+
+class Crawl4aiDockerClient:
+    """Client for interacting with Crawl4AI Docker server with token authentication."""
+    
+    def __init__(
+        self,
+        base_url: str = "http://localhost:8000",
+        timeout: float = 30.0,
+        verify_ssl: bool = True,
+        verbose: bool = True,
+        log_file: Optional[str] = None
+    ):
+        self.base_url = base_url.rstrip('/')
+        self.timeout = timeout
+        self.logger = AsyncLogger(log_file=log_file, log_level=LogLevel.DEBUG, verbose=verbose)
+        self._http_client = httpx.AsyncClient(
+            timeout=timeout,
+            verify=verify_ssl,
+            headers={"Content-Type": "application/json"}
+        )
+        self._token: Optional[str] = None
+
+    async def authenticate(self, email: str) -> None:
+        """Authenticate with the server and store the token."""
+        url = urljoin(self.base_url, "/token")
+        try:
+            self.logger.info(f"Authenticating with email: {email}", tag="AUTH")
+            response = await self._http_client.post(url, json={"email": email})
+            response.raise_for_status()
+            data = response.json()
+            self._token = data["access_token"]
+            self._http_client.headers["Authorization"] = f"Bearer {self._token}"
+            self.logger.success("Authentication successful", tag="AUTH")
+        except (httpx.RequestError, httpx.HTTPStatusError) as e:
+            error_msg = f"Authentication failed: {str(e)}"
+            self.logger.error(error_msg, tag="ERROR")
+            raise ConnectionError(error_msg)
+
+    async def _check_server(self) -> None:
+        """Check if server is reachable, raising an error if not."""
+        try:
+            await self._http_client.get(urljoin(self.base_url, "/health"))
+            self.logger.success(f"Connected to {self.base_url}", tag="READY")
+        except httpx.RequestError as e:
+            self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR")
+            raise ConnectionError(f"Cannot connect to server: {str(e)}")
+
+    def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, 
+                       crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]:
+        """Prepare request data from configs."""
+        return {
+            "urls": urls,
+            "browser_config": browser_config.dump() if browser_config else {},
+            "crawler_config": crawler_config.dump() if crawler_config else {}
+        }
+
+    async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
+        """Make an HTTP request with error handling."""
+        url = urljoin(self.base_url, endpoint)
+        try:
+            response = await self._http_client.request(method, url, **kwargs)
+            response.raise_for_status()
+            return response
+        except httpx.TimeoutException as e:
+            raise ConnectionError(f"Request timed out: {str(e)}")
+        except httpx.RequestError as e:
+            raise ConnectionError(f"Failed to connect: {str(e)}")
+        except httpx.HTTPStatusError as e:
+            error_msg = (e.response.json().get("detail", str(e)) 
+                        if "application/json" in e.response.headers.get("content-type", "") 
+                        else str(e))
+            raise RequestError(f"Server error {e.response.status_code}: {error_msg}")
+
+    async def crawl(
+        self,
+        urls: List[str],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None
+    ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
+        """Execute a crawl operation."""
+        if not self._token:
+            raise Crawl4aiClientError("Authentication required. Call authenticate() first.")
+        await self._check_server()
+        
+        data = self._prepare_request(urls, browser_config, crawler_config)
+        is_streaming = crawler_config and crawler_config.stream
+        
+        self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL")
+        
+        if is_streaming:
+            async def stream_results() -> AsyncGenerator[CrawlResult, None]:
+                async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response:
+                    response.raise_for_status()
+                    async for line in response.aiter_lines():
+                        if line.strip():
+                            result = json.loads(line)
+                            if "error" in result:
+                                self.logger.error_status(url=result.get("url", "unknown"), error=result["error"])
+                                continue
+                            self.logger.url_status(url=result.get("url", "unknown"), success=True, timing=result.get("timing", 0.0))
+                            if result.get("status") == "completed":
+                                continue
+                            else:
+                                yield CrawlResult(**result)
+            return stream_results()
+        
+        response = await self._request("POST", "/crawl", json=data)
+        result_data = response.json()
+        if not result_data.get("success", False):
+            raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}")
+        
+        results = [CrawlResult(**r) for r in result_data.get("results", [])]
+        self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL")
+        return results[0] if len(results) == 1 else results
+
+    async def get_schema(self) -> Dict[str, Any]:
+        """Retrieve configuration schemas."""
+        if not self._token:
+            raise Crawl4aiClientError("Authentication required. Call authenticate() first.")
+        response = await self._request("GET", "/schema")
+        return response.json()
+
+    async def close(self) -> None:
+        """Close the HTTP client session."""
+        self.logger.info("Closing client", tag="CLOSE")
+        await self._http_client.aclose()
+
+    async def __aenter__(self) -> "Crawl4aiDockerClient":
+        return self
+
+    async def __aexit__(self, exc_type: Optional[type], exc_val: Optional[Exception], exc_tb: Optional[Any]) -> None:
+        await self.close()
+
+
+# Example usage
+async def main():
+    async with Crawl4aiDockerClient(verbose=True) as client:
+        await client.authenticate("user@example.com")
+        result = await client.crawl(["https://example.com"])
+        print(result)
+        schema = await client.get_schema()
+        print(schema)
+
+if __name__ == "__main__":
+    asyncio.run(main())

crawl4ai/extraction_strategy.py

@@ -1,14 +1,13 @@
 from abc import ABC, abstractmethod
+import inspect
 from typing import Any, List, Dict, Optional
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import json
 import time
-import os
 
-from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH
+from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH, PROMPT_EXTRACT_INFERRED_SCHEMA
 from .config import (
-    DEFAULT_PROVIDER, PROVIDER_MODELS, 
-    CHUNK_TOKEN_THRESHOLD,
+    DEFAULT_PROVIDER, CHUNK_TOKEN_THRESHOLD,
     OVERLAP_RATE,
     WORD_TOKEN_RATE,
 )
@@ -21,6 +20,7 @@ from .utils import (
     extract_xml_data,
     split_and_parse_json_objects,
     sanitize_input_encode,
+    merge_chunks,
 )
 from .models import * # noqa: F403
 
@@ -34,8 +34,9 @@ from .model_loader import (
     calculate_batch_size
 )
 
+from .types import LLMConfig, create_llm_config
+
 from functools import partial
-import math
 import numpy as np
 import re
 from bs4 import BeautifulSoup
@@ -477,8 +478,7 @@ class LLMExtractionStrategy(ExtractionStrategy):
     A strategy that uses an LLM to extract meaningful content from the HTML.
 
     Attributes:
-        provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
-        api_token: The API token for the provider.
+        llm_config: The LLM configuration object.
         instruction: The instruction to use for the LLM model.
         schema: Pydantic model schema for structured data.
         extraction_type: "block" or "schema".
@@ -486,29 +486,41 @@ class LLMExtractionStrategy(ExtractionStrategy):
         overlap_rate: Overlap between chunks.
         word_token_rate: Word to token conversion rate.
         apply_chunking: Whether to apply chunking.
-        base_url: The base URL for the API request.
-        api_base: The base URL for the API request.
-        extra_args: Additional arguments for the API request, such as temprature, max_tokens, etc.
         verbose: Whether to print verbose output.
         usages: List of individual token usages.
         total_usage: Accumulated token usage.
     """
-
+    _UNWANTED_PROPS = {
+            'provider' : 'Instead, use llm_config=LLMConfig(provider="...")',
+            'api_token' : 'Instead, use llm_config=LlMConfig(api_token="...")',
+            'base_url' : 'Instead, use llm_config=LLMConfig(base_url="...")',
+            'api_base' : 'Instead, use llm_config=LLMConfig(base_url="...")',
+        }
     def __init__(
         self,
-        provider: str = DEFAULT_PROVIDER,
-        api_token: Optional[str] = None,
+        llm_config: 'LLMConfig' = None,
         instruction: str = None,
         schema: Dict = None,
         extraction_type="block",
+        chunk_token_threshold=CHUNK_TOKEN_THRESHOLD,
+        overlap_rate=OVERLAP_RATE,
+        word_token_rate=WORD_TOKEN_RATE,
+        apply_chunking=True,
+        input_format: str = "markdown",
+        force_json_response=False,
+        verbose=False,
+        # Deprecated arguments
+        provider: str = DEFAULT_PROVIDER,
+        api_token: Optional[str] = None,
+        base_url: str = None,
+        api_base: str = None,
         **kwargs,
     ):
         """
         Initialize the strategy with clustering parameters.
 
         Args:
-            provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
-            api_token: The API token for the provider.
+            llm_config: The LLM configuration object.
             instruction: The instruction to use for the LLM model.
             schema: Pydantic model schema for structured data.
             extraction_type: "block" or "schema".
@@ -516,48 +528,54 @@ class LLMExtractionStrategy(ExtractionStrategy):
             overlap_rate: Overlap between chunks.
             word_token_rate: Word to token conversion rate.
             apply_chunking: Whether to apply chunking.
+            input_format: Content format to use for extraction.
+                            Options: "markdown" (default), "html", "fit_markdown"
+            force_json_response: Whether to force a JSON response from the LLM.
+            verbose: Whether to print verbose output.
+
+            # Deprecated arguments, will be removed very soon
+            provider: The provider to use for extraction. It follows the format <provider_name>/<model_name>, e.g., "ollama/llama3.3".
+            api_token: The API token for the provider.
             base_url: The base URL for the API request.
             api_base: The base URL for the API request.
             extra_args: Additional arguments for the API request, such as temprature, max_tokens, etc.
-            verbose: Whether to print verbose output.
-            usages: List of individual token usages.
-            total_usage: Accumulated token usage.
-
         """
-        super().__init__(**kwargs)
-        self.provider = provider
-        self.api_token = (
-            api_token
-            or PROVIDER_MODELS.get(provider, "no-token")
-            or os.getenv("OPENAI_API_KEY")
-        )
+        super().__init__( input_format=input_format, **kwargs)
+        self.llm_config = llm_config
         self.instruction = instruction
         self.extract_type = extraction_type
         self.schema = schema
         if schema:
             self.extract_type = "schema"
-
-        self.chunk_token_threshold = kwargs.get(
-            "chunk_token_threshold", CHUNK_TOKEN_THRESHOLD
-        )
-        self.overlap_rate = kwargs.get("overlap_rate", OVERLAP_RATE)
-        self.word_token_rate = kwargs.get("word_token_rate", WORD_TOKEN_RATE)
-        self.apply_chunking = kwargs.get("apply_chunking", True)
-        self.base_url = kwargs.get("base_url", None)
-        self.api_base = kwargs.get("api_base", kwargs.get("base_url", None))
+        self.force_json_response = force_json_response
+        self.chunk_token_threshold = chunk_token_threshold or CHUNK_TOKEN_THRESHOLD
+        self.overlap_rate = overlap_rate
+        self.word_token_rate = word_token_rate
+        self.apply_chunking = apply_chunking
         self.extra_args = kwargs.get("extra_args", {})
         if not self.apply_chunking:
             self.chunk_token_threshold = 1e9
-
-        self.verbose = kwargs.get("verbose", False)
+        self.verbose = verbose
         self.usages = []  # Store individual usages
         self.total_usage = TokenUsage()  # Accumulated usage
 
-        if not self.api_token:
-            raise ValueError(
-                "API token must be provided for LLMExtractionStrategy. Update the config.py or set OPENAI_API_KEY environment variable."
-            )
+        self.provider = provider
+        self.api_token = api_token
+        self.base_url = base_url
+        self.api_base = api_base
 
+    
+    def __setattr__(self, name, value):
+        """Handle attribute setting."""
+        # TODO: Planning to set properties dynamically based on the __init__ signature
+        sig = inspect.signature(self.__init__)
+        all_params = sig.parameters  # Dictionary of parameter names and their details
+
+        if name in self._UNWANTED_PROPS and value is not all_params[name].default:
+            raise AttributeError(f"Setting '{name}' is deprecated. {self._UNWANTED_PROPS[name]}")
+        
+        super().__setattr__(name, value)  
+        
     def extract(self, url: str, ix: int, html: str) -> List[Dict[str, Any]]:
         """
         Extract meaningful blocks or chunks from the given HTML using an LLM.
@@ -590,115 +608,111 @@ class LLMExtractionStrategy(ExtractionStrategy):
             prompt_with_variables = PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION
 
         if self.extract_type == "schema" and self.schema:
-            variable_values["SCHEMA"] = json.dumps(self.schema, indent=2)
+            variable_values["SCHEMA"] = json.dumps(self.schema, indent=2) # if type of self.schema is dict else self.schema
             prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
 
+        if self.extract_type == "schema" and not self.schema:
+            prompt_with_variables = PROMPT_EXTRACT_INFERRED_SCHEMA
+
         for variable in variable_values:
             prompt_with_variables = prompt_with_variables.replace(
                 "{" + variable + "}", variable_values[variable]
             )
 
-        response = perform_completion_with_backoff(
-            self.provider,
-            prompt_with_variables,
-            self.api_token,
-            base_url=self.api_base or self.base_url,
-            extra_args=self.extra_args,
-        )  # , json_response=self.extract_type == "schema")
-        # Track usage
-        usage = TokenUsage(
-            completion_tokens=response.usage.completion_tokens,
-            prompt_tokens=response.usage.prompt_tokens,
-            total_tokens=response.usage.total_tokens,
-            completion_tokens_details=response.usage.completion_tokens_details.__dict__
-            if response.usage.completion_tokens_details
-            else {},
-            prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
-            if response.usage.prompt_tokens_details
-            else {},
-        )
-        self.usages.append(usage)
-
-        # Update totals
-        self.total_usage.completion_tokens += usage.completion_tokens
-        self.total_usage.prompt_tokens += usage.prompt_tokens
-        self.total_usage.total_tokens += usage.total_tokens
-
         try:
-            blocks = extract_xml_data(["blocks"], response.choices[0].message.content)[
-                "blocks"
-            ]
-            blocks = json.loads(blocks)
-            for block in blocks:
-                block["error"] = False
-        except Exception:
-            parsed, unparsed = split_and_parse_json_objects(
-                response.choices[0].message.content
+            response = perform_completion_with_backoff(
+                self.llm_config.provider,
+                prompt_with_variables,
+                self.llm_config.api_token,
+                base_url=self.llm_config.base_url,
+                json_response=self.force_json_response,
+                extra_args=self.extra_args,
+            )  # , json_response=self.extract_type == "schema")
+            # Track usage
+            usage = TokenUsage(
+                completion_tokens=response.usage.completion_tokens,
+                prompt_tokens=response.usage.prompt_tokens,
+                total_tokens=response.usage.total_tokens,
+                completion_tokens_details=response.usage.completion_tokens_details.__dict__
+                if response.usage.completion_tokens_details
+                else {},
+                prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
+                if response.usage.prompt_tokens_details
+                else {},
             )
-            blocks = parsed
-            if unparsed:
-                blocks.append(
-                    {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
+            self.usages.append(usage)
+
+            # Update totals
+            self.total_usage.completion_tokens += usage.completion_tokens
+            self.total_usage.prompt_tokens += usage.prompt_tokens
+            self.total_usage.total_tokens += usage.total_tokens
+
+            try:
+                response = response.choices[0].message.content
+                blocks = None
+
+                if self.force_json_response:
+                    blocks = json.loads(response)
+                    if isinstance(blocks, dict):
+                        # If it has only one key which calue is list then assign that to blocks, exampled: {"news": [..]}
+                        if len(blocks) == 1 and isinstance(list(blocks.values())[0], list):
+                            blocks = list(blocks.values())[0]
+                        else:
+                            # If it has only one key which value is not list then assign that to blocks, exampled: { "article_id": "1234", ... }
+                            blocks = [blocks]
+                    elif isinstance(blocks, list):
+                        # If it is a list then assign that to blocks
+                        blocks = blocks
+                else: 
+                    # blocks = extract_xml_data(["blocks"], response.choices[0].message.content)["blocks"]
+                    blocks = extract_xml_data(["blocks"], response)["blocks"]
+                    blocks = json.loads(blocks)
+
+                for block in blocks:
+                    block["error"] = False
+            except Exception:
+                parsed, unparsed = split_and_parse_json_objects(
+                    response
                 )
+                blocks = parsed
+                if unparsed:
+                    blocks.append(
+                        {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
+                    )
 
-        if self.verbose:
-            print(
-                "[LOG] Extracted",
-                len(blocks),
-                "blocks from URL:",
-                url,
-                "block index:",
-                ix,
-            )
-        return blocks
+            if self.verbose:
+                print(
+                    "[LOG] Extracted",
+                    len(blocks),
+                    "blocks from URL:",
+                    url,
+                    "block index:",
+                    ix,
+                )
+            return blocks
+        except Exception as e:
+            if self.verbose:
+                print(f"[LOG] Error in LLM extraction: {e}")
+            # Add error information to extracted_content
+            return [
+                {
+                    "index": ix,
+                    "error": True,
+                    "tags": ["error"],
+                    "content": str(e),
+                }
+            ]
 
-    def _merge(self, documents, chunk_token_threshold, overlap):
+    def _merge(self, documents, chunk_token_threshold, overlap) -> List[str]:
         """
         Merge documents into sections based on chunk_token_threshold and overlap.
         """
-        # chunks = []
-        sections = []
-        total_tokens = 0
-
-        # Calculate the total tokens across all documents
-        for document in documents:
-            total_tokens += len(document.split(" ")) * self.word_token_rate
-
-        # Calculate the number of sections needed
-        num_sections = math.floor(total_tokens / chunk_token_threshold)
-        if num_sections < 1:
-            num_sections = 1  # Ensure there is at least one section
-        adjusted_chunk_threshold = total_tokens / num_sections
-
-        total_token_so_far = 0
-        current_chunk = []
-
-        for document in documents:
-            tokens = document.split(" ")
-            token_count = len(tokens) * self.word_token_rate
-
-            if total_token_so_far + token_count <= adjusted_chunk_threshold:
-                current_chunk.extend(tokens)
-                total_token_so_far += token_count
-            else:
-                # Ensure to handle the last section properly
-                if len(sections) == num_sections - 1:
-                    current_chunk.extend(tokens)
-                    continue
-
-                # Add overlap if specified
-                if overlap > 0 and current_chunk:
-                    overlap_tokens = current_chunk[-overlap:]
-                    current_chunk.extend(overlap_tokens)
-
-                sections.append(" ".join(current_chunk))
-                current_chunk = tokens
-                total_token_so_far = token_count
-
-        # Add the last chunk
-        if current_chunk:
-            sections.append(" ".join(current_chunk))
-
+        sections =  merge_chunks(
+            docs = documents,
+            target_size= chunk_token_threshold,
+            overlap=overlap,
+            word_token_ratio=self.word_token_rate
+        )
         return sections
 
     def run(self, url: str, sections: List[str]) -> List[Dict[str, Any]]:
@@ -719,7 +733,7 @@ class LLMExtractionStrategy(ExtractionStrategy):
             overlap=int(self.chunk_token_threshold * self.overlap_rate),
         )
         extracted_content = []
-        if self.provider.startswith("groq/"):
+        if self.llm_config.provider.startswith("groq/"):
             # Sequential processing with a delay
             for ix, section in enumerate(merged_sections):
                 extract_func = partial(self.extract, url)
@@ -779,8 +793,6 @@ class LLMExtractionStrategy(ExtractionStrategy):
 #######################################################
 # New extraction strategies for JSON-based extraction #
 #######################################################
-
-
 class JsonElementExtractionStrategy(ExtractionStrategy):
     """
     Abstract base class for extracting structured JSON from HTML content.
@@ -1060,13 +1072,20 @@ class JsonElementExtractionStrategy(ExtractionStrategy):
         """Get attribute value from element"""
         pass
 
+    _GENERATE_SCHEMA_UNWANTED_PROPS = {
+        'provider': 'Instead, use llm_config=LLMConfig(provider="...")',
+        'api_token': 'Instead, use llm_config=LlMConfig(api_token="...")',
+    }
+
     @staticmethod
     def generate_schema(
         html: str,
         schema_type: str = "CSS", # or XPATH
         query: str = None,
-        provider: str = "gpt-4o",
-        api_token: str = os.getenv("OPENAI_API_KEY"),
+        target_json_example: str = None,
+        llm_config: 'LLMConfig' = create_llm_config(),
+        provider: str = None,
+        api_token: str = None,
         **kwargs
     ) -> dict:
         """
@@ -1075,16 +1094,20 @@ class JsonElementExtractionStrategy(ExtractionStrategy):
         Args:
             html (str): The HTML content to analyze
             query (str, optional): Natural language description of what data to extract
-            provider (str): LLM provider to use 
-            api_token (str): API token for LLM provider
+            provider (str): Legacy Parameter. LLM provider to use 
+            api_token (str): Legacy Parameter. API token for LLM provider
+            llm_config (LLMConfig): LLM configuration object
             prompt (str, optional): Custom prompt template to use
-            **kwargs: Additional args passed to perform_completion_with_backoff
+            **kwargs: Additional args passed to LLM processor
             
         Returns:
             dict: Generated schema following the JsonElementExtractionStrategy format
         """
         from .prompts import JSON_SCHEMA_BUILDER
         from .utils import perform_completion_with_backoff
+        for name, message in JsonElementExtractionStrategy._GENERATE_SCHEMA_UNWANTED_PROPS.items():
+            if locals()[name] is not None:
+                raise AttributeError(f"Setting '{name}' is deprecated. {message}")
         
         # Use default or custom prompt
         prompt_template = JSON_SCHEMA_BUILDER if schema_type == "CSS" else JSON_SCHEMA_BUILDER_XPATH
@@ -1092,32 +1115,65 @@ class JsonElementExtractionStrategy(ExtractionStrategy):
         # Build the prompt
         system_message = {
             "role": "system", 
-            "content": "You are a specialized HTML schema generator. Analyze the HTML and generate a JSON schema that follows the specified format. Only output valid JSON schema, nothing else."
+            "content": f"""You specialize in generating special JSON schemas for web scraping. This schema uses CSS or XPATH selectors to present a repetitive pattern in crawled HTML, such as a product in a product list or a search result item in a list of search results. We use this JSON schema to pass to a language model along with the HTML content to extract structured data from the HTML. The language model uses the JSON schema to extract data from the HTML and retrieve values for fields in the JSON schema, following the schema.
+
+Generating this HTML manually is not feasible, so you need to generate the JSON schema using the HTML content. The HTML copied from the crawled website is provided below, which we believe contains the repetitive pattern.
+
+# Schema main keys:
+- name: This is the name of the schema.
+- baseSelector: This is the CSS or XPATH selector that identifies the base element that contains all the repetitive patterns.
+- baseFields: This is a list of fields that you extract from the base element itself.
+- fields: This is a list of fields that you extract from the children of the base element. {{name, selector, type}} based on the type, you may have extra keys such as "attribute" when the type is "attribute".
+
+# Extra Context:
+In this context, the following items may or may not be present:
+- Example of target JSON object: This is a sample of the final JSON object that we hope to extract from the HTML using the schema you are generating.
+- Extra Instructions: This is optional instructions to consider when generating the schema provided by the user.
+- Query or explanation of target/goal data item: This is a description of what data we are trying to extract from the HTML. This explanation means we're not sure about the rigid schema of the structures we want, so we leave it to you to use your expertise to create the best and most comprehensive structures aimed at maximizing data extraction from this page. You must ensure that you do not pick up nuances that may exist on a particular page. The focus should be on the data we are extracting, and it must be valid, safe, and robust based on the given HTML.
+
+# What if there is no example of target JSON object and also no extra instructions or even no explanation of target/goal data item?
+In this scenario, use your best judgment to generate the schema. You need to examine the content of the page and understand the data it provides. If the page contains repetitive data, such as lists of items, products, jobs, places, books, or movies, focus on one single item that repeats. If the page is a detailed page about one product or item, create a schema to extract the entire structured data. At this stage, you must think and decide for yourself. Try to maximize the number of fields that you can extract from the HTML.
+
+# What are the instructions and details for this schema generation?
+{prompt_template}"""
         }
         
         user_message = {
             "role": "user",
             "content": f"""
-                Instructions:
-                {prompt_template}
-
                 HTML to analyze:
                 ```html
                 {html}
                 ```
-
-                {"Extract the following data: " + query if query else "Please analyze the HTML structure and create the most appropriate schema for data extraction."}
                 """
         }
 
+        if query:
+            user_message["content"] += f"\n\n## Query or explanation of target/goal data item:\n{query}"
+        if target_json_example:
+            user_message["content"] += f"\n\n## Example of target JSON object:\n```json\n{target_json_example}\n```"
+
+        if query and not target_json_example:
+            user_message["content"] += """IMPORTANT: To remind you, in this process, we are not providing a rigid example of the adjacent objects we seek. We rely on your understanding of the explanation provided in the above section. Make sure to grasp what we are looking for and, based on that, create the best schema.."""
+        elif not query and target_json_example:
+            user_message["content"] += """IMPORTANT: Please remember that in this process, we provided a proper example of a target JSON object. Make sure to adhere to the structure and create a schema that exactly fits this example. If you find that some elements on the page do not match completely, vote for the majority."""
+        elif not query and not target_json_example:
+            user_message["content"] += """IMPORTANT: Since we neither have a query nor an example, it is crucial to rely solely on the HTML content provided. Leverage your expertise to determine the schema based on the repetitive patterns observed in the content."""
+        
+        user_message["content"] += """IMPORTANT: Ensure your schema remains reliable by avoiding selectors that appear to generate dynamically and are not dependable. You want a reliable schema, as it consistently returns the same data even after many page reloads.
+
+        Analyze the HTML and generate a JSON schema that follows the specified format. Only output valid JSON schema, nothing else.
+        """
+
         try:
             # Call LLM with backoff handling
             response = perform_completion_with_backoff(
-                provider=provider,
+                provider=llm_config.provider,
                 prompt_with_variables="\n\n".join([system_message["content"], user_message["content"]]),
                 json_response = True,                
-                api_token=api_token,
-                **kwargs
+                api_token=llm_config.api_token,
+                base_url=llm_config.base_url,
+                extra_args=kwargs
             )
             
             # Extract and return schema
@@ -1126,7 +1182,6 @@ class JsonElementExtractionStrategy(ExtractionStrategy):
         except Exception as e:
             raise Exception(f"Failed to generate schema: {str(e)}")
 
-
 class JsonCssExtractionStrategy(JsonElementExtractionStrategy):
     """
     Concrete implementation of `JsonElementExtractionStrategy` using CSS selectors.
@@ -1154,7 +1209,8 @@ class JsonCssExtractionStrategy(JsonElementExtractionStrategy):
         super().__init__(schema, **kwargs)
 
     def _parse_html(self, html_content: str):
-        return BeautifulSoup(html_content, "html.parser")
+        # return BeautifulSoup(html_content, "html.parser")
+        return BeautifulSoup(html_content, "lxml")
 
     def _get_base_elements(self, parsed_html, selector: str):
         return parsed_html.select(selector)
@@ -1173,6 +1229,373 @@ class JsonCssExtractionStrategy(JsonElementExtractionStrategy):
     def _get_element_attribute(self, element, attribute: str):
         return element.get(attribute)
 
+class JsonLxmlExtractionStrategy(JsonElementExtractionStrategy):
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs["input_format"] = "html"
+        super().__init__(schema, **kwargs)
+        self._selector_cache = {}
+        self._xpath_cache = {}
+        self._result_cache = {}
+        
+        # Control selector optimization strategy
+        self.use_caching = kwargs.get("use_caching", True)
+        self.optimize_common_patterns = kwargs.get("optimize_common_patterns", True)
+        
+        # Load lxml dependencies once
+        from lxml import etree, html
+        from lxml.cssselect import CSSSelector
+        self.etree = etree
+        self.html_parser = html
+        self.CSSSelector = CSSSelector
+    
+    def _parse_html(self, html_content: str):
+        """Parse HTML content with error recovery"""
+        try:
+            parser = self.etree.HTMLParser(recover=True, remove_blank_text=True)
+            return self.etree.fromstring(html_content, parser)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error parsing HTML, falling back to alternative method: {e}")
+            try:
+                return self.html_parser.fromstring(html_content)
+            except Exception as e2:
+                if self.verbose:
+                    print(f"Critical error parsing HTML: {e2}")
+                # Create minimal document as fallback
+                return self.etree.Element("html")
+    
+    def _optimize_selector(self, selector_str):
+        """Optimize common selector patterns for better performance"""
+        if not self.optimize_common_patterns:
+            return selector_str
+            
+        # Handle td:nth-child(N) pattern which is very common in table scraping
+        import re
+        if re.search(r'td:nth-child\(\d+\)', selector_str):
+            return selector_str  # Already handled specially in _apply_selector
+            
+        # Split complex selectors into parts for optimization
+        parts = selector_str.split()
+        if len(parts) <= 1:
+            return selector_str
+            
+        # For very long selectors, consider using just the last specific part
+        if len(parts) > 3 and any(p.startswith('.') or p.startswith('#') for p in parts):
+            specific_parts = [p for p in parts if p.startswith('.') or p.startswith('#')]
+            if specific_parts:
+                return specific_parts[-1]  # Use most specific class/id selector
+                
+        return selector_str
+    
+    def _create_selector_function(self, selector_str):
+        """Create a selector function that handles all edge cases"""
+        original_selector = selector_str
+        
+        # Try to optimize the selector if appropriate
+        if self.optimize_common_patterns:
+            selector_str = self._optimize_selector(selector_str)
+        
+        try:
+            # Attempt to compile the CSS selector
+            compiled = self.CSSSelector(selector_str)
+            xpath = compiled.path
+            
+            # Store XPath for later use
+            self._xpath_cache[selector_str] = xpath
+            
+            # Create the wrapper function that implements the selection strategy
+            def selector_func(element, context_sensitive=True):
+                cache_key = None
+                
+                # Use result caching if enabled
+                if self.use_caching:
+                    # Create a cache key based on element and selector
+                    element_id = element.get('id', '') or str(hash(element))
+                    cache_key = f"{element_id}::{selector_str}"
+                    
+                    if cache_key in self._result_cache:
+                        return self._result_cache[cache_key]
+                
+                results = []
+                try:
+                    # Strategy 1: Direct CSS selector application (fastest)
+                    results = compiled(element)
+                    
+                    # If that fails and we need context sensitivity
+                    if not results and context_sensitive:
+                        # Strategy 2: Try XPath with context adjustment
+                        context_xpath = self._make_context_sensitive_xpath(xpath, element)
+                        if context_xpath:
+                            results = element.xpath(context_xpath)
+                        
+                        # Strategy 3: Handle special case - nth-child
+                        if not results and 'nth-child' in original_selector:
+                            results = self._handle_nth_child_selector(element, original_selector)
+                        
+                        # Strategy 4: Direct descendant search for class/ID selectors
+                        if not results:
+                            results = self._fallback_class_id_search(element, original_selector)
+                            
+                        # Strategy 5: Last resort - tag name search for the final part
+                        if not results:
+                            parts = original_selector.split()
+                            if parts:
+                                last_part = parts[-1]
+                                # Extract tag name from the selector
+                                tag_match = re.match(r'^(\w+)', last_part)
+                                if tag_match:
+                                    tag_name = tag_match.group(1)
+                                    results = element.xpath(f".//{tag_name}")
+                    
+                    # Cache results if caching is enabled
+                    if self.use_caching and cache_key:
+                        self._result_cache[cache_key] = results
+                        
+                except Exception as e:
+                    if self.verbose:
+                        print(f"Error applying selector '{selector_str}': {e}")
+                
+                return results
+                
+            return selector_func
+            
+        except Exception as e:
+            if self.verbose:
+                print(f"Error compiling selector '{selector_str}': {e}")
+            
+            # Fallback function for invalid selectors
+            return lambda element, context_sensitive=True: []
+    
+    def _make_context_sensitive_xpath(self, xpath, element):
+        """Convert absolute XPath to context-sensitive XPath"""
+        try:
+            # If starts with descendant-or-self, it's already context-sensitive
+            if xpath.startswith('descendant-or-self::'):
+                return xpath
+                
+            # Remove leading slash if present
+            if xpath.startswith('/'):
+                context_xpath = f".{xpath}"
+            else:
+                context_xpath = f".//{xpath}"
+                
+            # Validate the XPath by trying it
+            try:
+                element.xpath(context_xpath)
+                return context_xpath
+            except:
+                # If that fails, try a simpler descendant search
+                return f".//{xpath.split('/')[-1]}"
+        except:
+            return None
+    
+    def _handle_nth_child_selector(self, element, selector_str):
+        """Special handling for nth-child selectors in tables"""
+        import re
+        results = []
+        
+        try:
+            # Extract the column number from td:nth-child(N)
+            match = re.search(r'td:nth-child\((\d+)\)', selector_str)
+            if match:
+                col_num = match.group(1)
+                
+                # Check if there's content after the nth-child part
+                remaining_selector = selector_str.split(f"td:nth-child({col_num})", 1)[-1].strip()
+                
+                if remaining_selector:
+                    # If there's a specific element we're looking for after the column
+                    # Extract any tag names from the remaining selector
+                    tag_match = re.search(r'(\w+)', remaining_selector)
+                    tag_name = tag_match.group(1) if tag_match else '*'
+                    results = element.xpath(f".//td[{col_num}]//{tag_name}")
+                else:
+                    # Just get the column cell
+                    results = element.xpath(f".//td[{col_num}]")
+        except Exception as e:
+            if self.verbose:
+                print(f"Error handling nth-child selector: {e}")
+                
+        return results
+    
+    def _fallback_class_id_search(self, element, selector_str):
+        """Fallback to search by class or ID"""
+        results = []
+        
+        try:
+            # Extract class selectors (.classname)
+            import re
+            class_matches = re.findall(r'\.([a-zA-Z0-9_-]+)', selector_str)
+            
+            # Extract ID selectors (#idname)
+            id_matches = re.findall(r'#([a-zA-Z0-9_-]+)', selector_str)
+            
+            # Try each class
+            for class_name in class_matches:
+                class_results = element.xpath(f".//*[contains(@class, '{class_name}')]")
+                results.extend(class_results)
+                
+            # Try each ID (usually more specific)
+            for id_name in id_matches:
+                id_results = element.xpath(f".//*[@id='{id_name}']")
+                results.extend(id_results)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error in fallback class/id search: {e}")
+                
+        return results
+    
+    def _get_selector(self, selector_str):
+        """Get or create a selector function with caching"""
+        if selector_str not in self._selector_cache:
+            self._selector_cache[selector_str] = self._create_selector_function(selector_str)
+        return self._selector_cache[selector_str]
+    
+    def _get_base_elements(self, parsed_html, selector: str):
+        """Get all base elements using the selector"""
+        selector_func = self._get_selector(selector)
+        # For base elements, we don't need context sensitivity
+        return selector_func(parsed_html, context_sensitive=False)
+    
+    def _get_elements(self, element, selector: str):
+        """Get child elements using the selector with context sensitivity"""
+        selector_func = self._get_selector(selector)
+        return selector_func(element, context_sensitive=True)
+    
+    def _get_element_text(self, element) -> str:
+        """Extract normalized text from element"""
+        try:
+            # Get all text nodes and normalize
+            text = " ".join(t.strip() for t in element.xpath(".//text()") if t.strip())
+            return text
+        except Exception as e:
+            if self.verbose:
+                print(f"Error extracting text: {e}")
+            # Fallback
+            try:
+                return element.text_content().strip()
+            except:
+                return ""
+    
+    def _get_element_html(self, element) -> str:
+        """Get HTML string representation of element"""
+        try:
+            return self.etree.tostring(element, encoding='unicode', method='html')
+        except Exception as e:
+            if self.verbose:
+                print(f"Error serializing HTML: {e}")
+            return ""
+    
+    def _get_element_attribute(self, element, attribute: str):
+        """Get attribute value safely"""
+        try:
+            return element.get(attribute)
+        except Exception as e:
+            if self.verbose:
+                print(f"Error getting attribute '{attribute}': {e}")
+            return None
+            
+    def _clear_caches(self):
+        """Clear caches to free memory"""
+        if self.use_caching:
+            self._result_cache.clear()
+
+class JsonLxmlExtractionStrategy_naive(JsonElementExtractionStrategy):
+    def __init__(self, schema: Dict[str, Any], **kwargs):
+        kwargs["input_format"] = "html"  # Force HTML input
+        super().__init__(schema, **kwargs)
+        self._selector_cache = {}
+    
+    def _parse_html(self, html_content: str):
+        from lxml import etree
+        parser = etree.HTMLParser(recover=True)
+        return etree.fromstring(html_content, parser)
+    
+    def _get_selector(self, selector_str):
+        """Get a selector function that works within the context of an element"""
+        if selector_str not in self._selector_cache:
+            from lxml.cssselect import CSSSelector
+            try:
+                # Store both the compiled selector and its xpath translation
+                compiled = CSSSelector(selector_str)
+                
+                # Create a function that will apply this selector appropriately
+                def select_func(element):
+                    try:
+                        # First attempt: direct CSS selector application
+                        results = compiled(element)
+                        if results:
+                            return results
+                        
+                        # Second attempt: contextual XPath selection
+                        # Convert the root-based XPath to a context-based XPath
+                        xpath = compiled.path
+                        
+                        # If the XPath already starts with descendant-or-self, handle it specially
+                        if xpath.startswith('descendant-or-self::'):
+                            context_xpath = xpath
+                        else:
+                            # For normal XPath expressions, make them relative to current context
+                            context_xpath = f"./{xpath.lstrip('/')}"
+                        
+                        results = element.xpath(context_xpath)
+                        if results:
+                            return results
+                        
+                        # Final fallback: simple descendant search for common patterns
+                        if 'nth-child' in selector_str:
+                            # Handle td:nth-child(N) pattern
+                            import re
+                            match = re.search(r'td:nth-child\((\d+)\)', selector_str)
+                            if match:
+                                col_num = match.group(1)
+                                sub_selector = selector_str.split(')', 1)[-1].strip()
+                                if sub_selector:
+                                    return element.xpath(f".//td[{col_num}]//{sub_selector}")
+                                else:
+                                    return element.xpath(f".//td[{col_num}]")
+                        
+                        # Last resort: try each part of the selector separately
+                        parts = selector_str.split()
+                        if len(parts) > 1 and parts[-1]:
+                            return element.xpath(f".//{parts[-1]}")
+                            
+                        return []
+                    except Exception as e:
+                        if self.verbose:
+                            print(f"Error applying selector '{selector_str}': {e}")
+                        return []
+                
+                self._selector_cache[selector_str] = select_func
+            except Exception as e:
+                if self.verbose:
+                    print(f"Error compiling selector '{selector_str}': {e}")
+                
+                # Fallback function for invalid selectors
+                def fallback_func(element):
+                    return []
+                
+                self._selector_cache[selector_str] = fallback_func
+                
+        return self._selector_cache[selector_str]
+    
+    def _get_base_elements(self, parsed_html, selector: str):
+        selector_func = self._get_selector(selector)
+        return selector_func(parsed_html)
+    
+    def _get_elements(self, element, selector: str):
+        selector_func = self._get_selector(selector)
+        return selector_func(element)
+    
+    def _get_element_text(self, element) -> str:
+        return "".join(element.xpath(".//text()")).strip()
+    
+    def _get_element_html(self, element) -> str:
+        from lxml import etree
+        return etree.tostring(element, encoding='unicode')
+    
+    def _get_element_attribute(self, element, attribute: str):
+        return element.get(attribute)    
 
 class JsonXPathExtractionStrategy(JsonElementExtractionStrategy):
     """

crawl4ai/html2text/__init__.py

@@ -510,6 +510,7 @@ class HTML2Text(html.parser.HTMLParser):
 
         if tag == "a" and not self.ignore_links:
             if start:
+                self.inside_link = True
                 if (
                     "href" in attrs
                     and attrs["href"] is not None
@@ -526,6 +527,7 @@ class HTML2Text(html.parser.HTMLParser):
                 else:
                     self.astack.append(None)
             else:
+                self.inside_link = False
                 if self.astack:
                     a = self.astack.pop()
                     if self.maybe_automatic_link and not self.empty_link:
@@ -610,13 +612,22 @@ class HTML2Text(html.parser.HTMLParser):
                         self.o("[" + str(a_props.count) + "]")
 
         if tag == "dl" and start:
-            self.p()
-        if tag == "dt" and not start:
-            self.pbr()
-        if tag == "dd" and start:
-            self.o("    ")
-        if tag == "dd" and not start:
-            self.pbr()
+            self.p()  # Add paragraph break before list starts
+            self.p_p = 0  # Reset paragraph state
+        
+        elif tag == "dt" and start:
+            if self.p_p == 0:  # If not first term
+                self.o("\n\n")  # Add spacing before new term-definition pair
+            self.p_p = 0  # Reset paragraph state
+        
+        elif tag == "dt" and not start:
+            self.o("\n")  # Single newline between term and definition
+        
+        elif tag == "dd" and start:
+            self.o("    ")  # Indent definition
+        
+        elif tag == "dd" and not start:
+            self.p_p = 0
 
         if tag in ["ol", "ul"]:
             # Google Docs create sub lists as top level lists
@@ -1026,6 +1037,7 @@ class CustomHTML2Text(HTML2Text):
         super().__init__(*args, **kwargs)
         self.inside_pre = False
         self.inside_code = False
+        self.inside_link = False
         self.preserve_tags = set()  # Set of tags to preserve
         self.current_preserved_tag = None
         self.preserved_content = []
@@ -1105,11 +1117,17 @@ class CustomHTML2Text(HTML2Text):
                 # Ignore code tags inside pre blocks if handle_code_in_pre is False
                 return
             if start:
-                self.o("`")  # Markdown inline code start
+                if not self.inside_link:
+                    self.o("`")  # Only output backtick if not inside a link
                 self.inside_code = True
             else:
-                self.o("`")  # Markdown inline code end
+                if not self.inside_link:
+                    self.o("`")  # Only output backtick if not inside a link
                 self.inside_code = False
+
+            # If inside a link, let the parent class handle the content
+            if self.inside_link:
+                super().handle_tag(tag, attrs, start) 
         else:
             super().handle_tag(tag, attrs, start)
 

crawl4ai/hub.py

@@ -0,0 +1,69 @@
+# crawl4ai/hub.py
+from abc import ABC, abstractmethod
+from typing import Dict, Type, Union
+import logging
+import importlib
+from pathlib import Path
+import inspect
+
+logger = logging.getLogger(__name__)
+
+
+class BaseCrawler(ABC):
+    def __init__(self):
+        self.logger = logging.getLogger(self.__class__.__name__)
+        
+    @abstractmethod
+    async def run(self, url: str = "", **kwargs) -> str:
+        """
+        Implement this method to return JSON string.
+        Must accept URL + arbitrary kwargs for flexibility.
+        """
+        pass
+
+    def __init_subclass__(cls, **kwargs):
+        """Enforce interface validation on subclassing"""
+        super().__init_subclass__(**kwargs)
+        
+        # Verify run method signature
+        run_method = cls.run
+        if not run_method.__code__.co_argcount >= 2:  # self + url
+            raise TypeError(f"{cls.__name__} must implement 'run(self, url: str, **kwargs)'")
+            
+        # Verify async nature
+        if not inspect.iscoroutinefunction(run_method):
+            raise TypeError(f"{cls.__name__}.run must be async")
+
+class CrawlerHub:
+    _crawlers: Dict[str, Type[BaseCrawler]] = {}
+
+    @classmethod
+    def _discover_crawlers(cls):
+        """Dynamically load crawlers from /crawlers in 3 lines"""
+        base_path = Path(__file__).parent / "crawlers"
+        for crawler_dir in base_path.iterdir():
+            if crawler_dir.is_dir():
+                try:
+                    module = importlib.import_module(
+                        f"crawl4ai.crawlers.{crawler_dir.name}.crawler"
+                    )
+                    for attr in dir(module):
+                        cls._maybe_register_crawler(
+                            getattr(module, attr), crawler_dir.name
+                        )
+                except Exception as e:
+                    logger.warning(f"Failed {crawler_dir.name}: {str(e)}")
+
+    @classmethod
+    def _maybe_register_crawler(cls, obj, name: str):
+        """Brilliant one-liner registration"""
+        if isinstance(obj, type) and issubclass(obj, BaseCrawler) and obj != BaseCrawler:
+            module = importlib.import_module(obj.__module__)
+            obj.meta = getattr(module, "__meta__", {})
+            cls._crawlers[name] = obj
+
+    @classmethod
+    def get(cls, name: str) -> Union[Type[BaseCrawler], None]:
+        if not cls._crawlers:
+            cls._discover_crawlers()
+        return cls._crawlers.get(name)

crawl4ai/install.py

@@ -2,17 +2,78 @@ import subprocess
 import sys
 import asyncio
 from .async_logger import AsyncLogger, LogLevel
+from pathlib import Path
+import os
+import shutil
 
 # Initialize logger
 logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
 
+def setup_home_directory():
+    """Set up the .crawl4ai folder structure in the user's home directory."""
+    base_dir = os.getenv("CRAWL4_AI_BASE_DIRECTORY")
+    crawl4ai_folder = Path(base_dir) if base_dir else Path.home()
+    crawl4ai_config = crawl4ai_folder / "global.yml"
+    crawl4ai_folder = crawl4ai_folder / ".crawl4ai"
+    cache_folder = crawl4ai_folder / "cache"
+    content_folders = [
+        "html_content",
+        "cleaned_html",
+        "markdown_content",
+        "extracted_content",
+        "screenshots",
+    ]
+
+    # Clean up old cache if exists
+    if cache_folder.exists():
+        shutil.rmtree(cache_folder)
+
+    # Create new folder structure
+    crawl4ai_folder.mkdir(exist_ok=True)
+    cache_folder.mkdir(exist_ok=True)
+    for folder in content_folders:
+        (crawl4ai_folder / folder).mkdir(exist_ok=True)
+    
+    # If config file does not exist, create it
+    if not crawl4ai_config.exists():
+        with open(crawl4ai_config, "w") as f:
+            f.write("")
 
 def post_install():
     """Run all post-installation tasks"""
     logger.info("Running post-installation setup...", tag="INIT")
+    setup_home_directory()
     install_playwright()
     run_migration()
+    # TODO: Will be added in the future
+    # setup_builtin_browser()
     logger.success("Post-installation setup completed!", tag="COMPLETE")
+    
+def setup_builtin_browser():
+    """Set up a builtin browser for use with Crawl4AI"""
+    try:
+        logger.info("Setting up builtin browser...", tag="INIT")
+        asyncio.run(_setup_builtin_browser())
+        logger.success("Builtin browser setup completed!", tag="COMPLETE")
+    except Exception as e:
+        logger.warning(f"Failed to set up builtin browser: {e}")
+        logger.warning("You can manually set up a builtin browser using 'crawl4ai-doctor builtin-browser-start'")
+    
+async def _setup_builtin_browser():
+    try:
+        # Import BrowserProfiler here to avoid circular imports
+        from .browser_profiler import BrowserProfiler
+        profiler = BrowserProfiler(logger=logger)
+        
+        # Launch the builtin browser
+        cdp_url = await profiler.launch_builtin_browser(headless=True)
+        if cdp_url:
+            logger.success(f"Builtin browser launched at {cdp_url}", tag="BROWSER")
+        else:
+            logger.warning("Failed to launch builtin browser", tag="BROWSER")
+    except Exception as e:
+        logger.warning(f"Error setting up builtin browser: {e}", tag="BROWSER")
+        raise
 
 
 def install_playwright():
@@ -106,4 +167,5 @@ def doctor():
     """Entry point for the doctor command"""
     import asyncio
 
-    return asyncio.run(run_doctor())
+    asyncio.run(run_doctor())
+    sys.exit(0)

crawl4ai/legacy/cli.py

@@ -0,0 +1,123 @@
+import click
+import sys
+import asyncio
+from typing import List
+from .docs_manager import DocsManager
+from .async_logger import AsyncLogger
+
+logger = AsyncLogger(verbose=True)
+docs_manager = DocsManager(logger)
+
+
+def print_table(headers: List[str], rows: List[List[str]], padding: int = 2):
+    """Print formatted table with headers and rows"""
+    widths = [max(len(str(cell)) for cell in col) for col in zip(headers, *rows)]
+    border = "+" + "+".join("-" * (w + 2 * padding) for w in widths) + "+"
+
+    def format_row(row):
+        return (
+            "|"
+            + "|".join(
+                f"{' ' * padding}{str(cell):<{w}}{' ' * padding}"
+                for cell, w in zip(row, widths)
+            )
+            + "|"
+        )
+
+    click.echo(border)
+    click.echo(format_row(headers))
+    click.echo(border)
+    for row in rows:
+        click.echo(format_row(row))
+    click.echo(border)
+
+
+@click.group()
+def cli():
+    """Crawl4AI Command Line Interface"""
+    pass
+
+
+@cli.group()
+def docs():
+    """Documentation operations"""
+    pass
+
+
+@docs.command()
+@click.argument("sections", nargs=-1)
+@click.option(
+    "--mode", type=click.Choice(["extended", "condensed"]), default="extended"
+)
+def combine(sections: tuple, mode: str):
+    """Combine documentation sections"""
+    try:
+        asyncio.run(docs_manager.ensure_docs_exist())
+        click.echo(docs_manager.generate(sections, mode))
+    except Exception as e:
+        logger.error(str(e), tag="ERROR")
+        sys.exit(1)
+
+
+@docs.command()
+@click.argument("query")
+@click.option("--top-k", "-k", default=5)
+@click.option("--build-index", is_flag=True, help="Build index if missing")
+def search(query: str, top_k: int, build_index: bool):
+    """Search documentation"""
+    try:
+        result = docs_manager.search(query, top_k)
+        if result == "No search index available. Call build_search_index() first.":
+            if build_index or click.confirm("No search index found. Build it now?"):
+                asyncio.run(docs_manager.llm_text.generate_index_files())
+                result = docs_manager.search(query, top_k)
+        click.echo(result)
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+@docs.command()
+def update():
+    """Update docs from GitHub"""
+    try:
+        asyncio.run(docs_manager.fetch_docs())
+        click.echo("Documentation updated successfully")
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+@docs.command()
+@click.option("--force-facts", is_flag=True, help="Force regenerate fact files")
+@click.option("--clear-cache", is_flag=True, help="Clear BM25 cache")
+def index(force_facts: bool, clear_cache: bool):
+    """Build or rebuild search indexes"""
+    try:
+        asyncio.run(docs_manager.ensure_docs_exist())
+        asyncio.run(
+            docs_manager.llm_text.generate_index_files(
+                force_generate_facts=force_facts, clear_bm25_cache=clear_cache
+            )
+        )
+        click.echo("Search indexes built successfully")
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+# Add docs list command
+@docs.command()
+def list():
+    """List available documentation sections"""
+    try:
+        sections = docs_manager.list()
+        print_table(["Sections"], [[section] for section in sections])
+
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    cli()

crawl4ai/markdown_generation_strategy.py

@@ -2,6 +2,7 @@ from abc import ABC, abstractmethod
 from typing import Optional, Dict, Any, Tuple
 from .models import MarkdownGenerationResult
 from .html2text import CustomHTML2Text
+# from .types import RelevantContentFilter
 from .content_filter_strategy import RelevantContentFilter
 import re
 from urllib.parse import urljoin
@@ -29,9 +30,11 @@ class MarkdownGenerationStrategy(ABC):
         self,
         content_filter: Optional[RelevantContentFilter] = None,
         options: Optional[Dict[str, Any]] = None,
+        verbose: bool = False,
     ):
         self.content_filter = content_filter
         self.options = options or {}
+        self.verbose = verbose
 
     @abstractmethod
     def generate_markdown(
@@ -176,7 +179,7 @@ class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
                 "ignore_emphasis": False,
                 "ignore_links": False,
                 "ignore_images": False,
-                "protect_links": True,
+                "protect_links": False,
                 "single_line_break": True,
                 "mark_code": True,
                 "escape_snob": False,

crawl4ai/models.py

@@ -1,5 +1,7 @@
-from pydantic import BaseModel, HttpUrl
+from pydantic import BaseModel, HttpUrl, PrivateAttr, ConfigDict
 from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
+from typing import AsyncGenerator
+from typing import Generic, TypeVar
 from enum import Enum
 from dataclasses import dataclass
 from .ssl_certificate import SSLCertificate
@@ -24,10 +26,15 @@ class CrawlerTaskResult:
     result: "CrawlResult"
     memory_usage: float
     peak_memory: float
-    start_time: datetime
-    end_time: datetime
+    start_time: Union[datetime, float]
+    end_time: Union[datetime, float]
     error_message: str = ""
-
+    retry_count: int = 0
+    wait_time: float = 0.0
+    
+    @property
+    def success(self) -> bool:
+        return self.result.success
 
 class CrawlStatus(Enum):
     QUEUED = "QUEUED"
@@ -35,27 +42,39 @@ class CrawlStatus(Enum):
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
 
-
 @dataclass
 class CrawlStats:
     task_id: str
     url: str
     status: CrawlStatus
-    start_time: Optional[datetime] = None
-    end_time: Optional[datetime] = None
+    start_time: Optional[Union[datetime, float]] = None
+    end_time: Optional[Union[datetime, float]] = None
     memory_usage: float = 0.0
     peak_memory: float = 0.0
     error_message: str = ""
+    wait_time: float = 0.0
+    retry_count: int = 0
+    counted_requeue: bool = False
 
     @property
     def duration(self) -> str:
         if not self.start_time:
             return "0:00"
+            
+        # Convert start_time to datetime if it's a float
+        start = self.start_time
+        if isinstance(start, float):
+            start = datetime.fromtimestamp(start)
+            
+        # Get end time or use current time
         end = self.end_time or datetime.now()
-        duration = end - self.start_time
+        # Convert end_time to datetime if it's a float
+        if isinstance(end, float):
+            end = datetime.fromtimestamp(end)
+            
+        duration = end - start
         return str(timedelta(seconds=int(duration.total_seconds())))
 
-
 class DisplayMode(Enum):
     DETAILED = "DETAILED"
     AGGREGATED = "AGGREGATED"
@@ -72,12 +91,10 @@ class TokenUsage:
     completion_tokens_details: Optional[dict] = None
     prompt_tokens_details: Optional[dict] = None
 
-
 class UrlModel(BaseModel):
     url: HttpUrl
     forced: bool = False
 
-
 class MarkdownGenerationResult(BaseModel):
     raw_markdown: str
     markdown_with_citations: str
@@ -85,16 +102,28 @@ class MarkdownGenerationResult(BaseModel):
     fit_markdown: Optional[str] = None
     fit_html: Optional[str] = None
 
+    def __str__(self):
+        return self.raw_markdown
+
+@dataclass
+class TraversalStats:
+    """Statistics for the traversal process"""
+
+    start_time: datetime = datetime.now()
+    urls_processed: int = 0
+    urls_failed: int = 0
+    urls_skipped: int = 0
+    total_depth_reached: int = 0
+    current_depth: int = 0
 
 class DispatchResult(BaseModel):
     task_id: str
     memory_usage: float
     peak_memory: float
-    start_time: datetime
-    end_time: datetime
+    start_time: Union[datetime, float]
+    end_time: Union[datetime, float]
     error_message: str = ""
 
-
 class CrawlResult(BaseModel):
     url: str
     html: str
@@ -103,12 +132,10 @@ class CrawlResult(BaseModel):
     media: Dict[str, List[Dict]] = {}
     links: Dict[str, List[Dict]] = {}
     downloaded_files: Optional[List[str]] = None
+    js_execution_result: Optional[Dict[str, Any]] = None
     screenshot: Optional[str] = None
     pdf: Optional[bytes] = None
-    markdown: Optional[Union[str, MarkdownGenerationResult]] = None
-    markdown_v2: Optional[MarkdownGenerationResult] = None
-    fit_markdown: Optional[str] = None
-    fit_html: Optional[str] = None
+    _markdown: Optional[MarkdownGenerationResult] = PrivateAttr(default=None)
     extracted_content: Optional[str] = None
     metadata: Optional[dict] = None
     error_message: Optional[str] = None
@@ -119,13 +146,165 @@ class CrawlResult(BaseModel):
     dispatch_result: Optional[DispatchResult] = None
     redirected_url: Optional[str] = None
 
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    # class Config:
+    #     arbitrary_types_allowed = True
 
+# NOTE: The StringCompatibleMarkdown class, custom __init__ method, property getters/setters,
+# and model_dump override all exist to support a smooth transition from markdown as a string
+# to markdown as a MarkdownGenerationResult object, while maintaining backward compatibility.
+# 
+# This allows code that expects markdown to be a string to continue working, while also
+# providing access to the full MarkdownGenerationResult object's properties.
+# 
+# The markdown_v2 property is deprecated and raises an error directing users to use markdown.
+# 
+# When backward compatibility is no longer needed in future versions, this entire mechanism
+# can be simplified to a standard field with no custom accessors or serialization logic.
+    
+    def __init__(self, **data):
+        markdown_result = data.pop('markdown', None)
+        super().__init__(**data)
+        if markdown_result is not None:
+            self._markdown = (
+                MarkdownGenerationResult(**markdown_result)
+                if isinstance(markdown_result, dict)
+                else markdown_result
+            )
+    
+    @property
+    def markdown(self):
+        """
+        Property that returns a StringCompatibleMarkdown object that behaves like
+        a string but also provides access to MarkdownGenerationResult attributes.
+        
+        This approach allows backward compatibility with code that expects 'markdown'
+        to be a string, while providing access to the full MarkdownGenerationResult.
+        """
+        if self._markdown is None:
+            return None
+        return StringCompatibleMarkdown(self._markdown)
+    
+    @markdown.setter
+    def markdown(self, value):
+        """
+        Setter for the markdown property.
+        """
+        self._markdown = value
+    
+    @property
+    def markdown_v2(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+
+        This property exists to inform users that 'markdown_v2' has been
+        deprecated and they should use 'markdown' instead.
+        """
+        raise AttributeError(
+            "The 'markdown_v2' attribute is deprecated and has been removed. "
+            """Please use 'markdown' instead, which now returns a MarkdownGenerationResult, with
+            following properties:
+            - raw_markdown: The raw markdown string
+            - markdown_with_citations: The markdown string with citations
+            - references_markdown: The markdown string with references
+            - fit_markdown: The markdown string with fit text
+            """
+        )
+    
+    @property
+    def fit_markdown(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+        """
+        raise AttributeError(
+            "The 'fit_markdown' attribute is deprecated and has been removed. "
+            "Please use 'markdown.fit_markdown' instead."
+        )
+    
+    @property
+    def fit_html(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+        """
+        raise AttributeError(
+            "The 'fit_html' attribute is deprecated and has been removed. "
+            "Please use 'markdown.fit_html' instead."
+        )
+
+    def model_dump(self, *args, **kwargs):
+        """
+        Override model_dump to include the _markdown private attribute in serialization.
+        
+        This override is necessary because:
+        1. PrivateAttr fields are excluded from serialization by default
+        2. We need to maintain backward compatibility by including the 'markdown' field
+           in the serialized output
+        3. We're transitioning from 'markdown_v2' to enhancing 'markdown' to hold
+           the same type of data
+        
+        Future developers: This method ensures that the markdown content is properly
+        serialized despite being stored in a private attribute. If the serialization
+        requirements change, this is where you would update the logic.
+        """
+        result = super().model_dump(*args, **kwargs)
+        if self._markdown is not None:
+            result["markdown"] = self._markdown.model_dump() 
+        return result
+
+class StringCompatibleMarkdown(str):
+    """A string subclass that also provides access to MarkdownGenerationResult attributes"""
+    def __new__(cls, markdown_result):
+        return super().__new__(cls, markdown_result.raw_markdown)
+    
+    def __init__(self, markdown_result):
+        self._markdown_result = markdown_result
+    
+    def __getattr__(self, name):
+        return getattr(self._markdown_result, name)
+
+CrawlResultT = TypeVar('CrawlResultT', bound=CrawlResult)
+
+class CrawlResultContainer(Generic[CrawlResultT]):
+    def __init__(self, results: Union[CrawlResultT, List[CrawlResultT]]):
+        # Normalize to a list
+        if isinstance(results, list):
+            self._results = results
+        else:
+            self._results = [results]
+
+    def __iter__(self):
+        return iter(self._results)
+
+    def __getitem__(self, index):
+        return self._results[index]
+
+    def __len__(self):
+        return len(self._results)
+
+    def __getattr__(self, attr):
+        # Delegate attribute access to the first element.
+        if self._results:
+            return getattr(self._results[0], attr)
+        raise AttributeError(f"{self.__class__.__name__} object has no attribute '{attr}'")
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._results!r})"
+
+RunManyReturn = Union[
+    CrawlResultContainer[CrawlResultT],
+    AsyncGenerator[CrawlResultT, None]
+]
+
+
+# END of backward compatibility code for markdown/markdown_v2.
+# When removing this code in the future, make sure to:
+# 1. Replace the private attribute and property with a standard field
+# 2. Update any serialization logic that might depend on the current behavior
 
 class AsyncCrawlResponse(BaseModel):
     html: str
     response_headers: Dict[str, str]
+    js_execution_result: Optional[Dict[str, Any]] = None
     status_code: int
     screenshot: Optional[str] = None
     pdf_data: Optional[bytes] = None
@@ -134,15 +313,16 @@ class AsyncCrawlResponse(BaseModel):
     ssl_certificate: Optional[SSLCertificate] = None
     redirected_url: Optional[str] = None
 
-    class Config:
-        arbitrary_types_allowed = True
-
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    # class Config:
+    #     arbitrary_types_allowed = True
 
 ###############################
 # Scraping Models
 ###############################
 class MediaItem(BaseModel):
     src: Optional[str] = ""
+    data: Optional[str] = ""
     alt: Optional[str] = ""
     desc: Optional[str] = ""
     score: Optional[int] = 0
@@ -167,6 +347,7 @@ class Media(BaseModel):
     audios: List[
         MediaItem
     ] = []  # Using MediaItem model for now, can be extended with Audio model if needed
+    tables: List[Dict] = []  # Table data extracted from HTML tables
 
 
 class Links(BaseModel):

crawl4ai/pipeline/__init__.py

@@ -0,0 +1,6 @@
+"""Pipeline module providing high-level crawling functionality."""
+
+from .pipeline import Pipeline, create_pipeline
+from .crawler import Crawler
+
+__all__ = ["Pipeline", "create_pipeline", "Crawler"]

crawl4ai/pipeline/crawler.py

@@ -0,0 +1,406 @@
+"""Crawler utility class for simplified crawling operations.
+
+This module provides a high-level utility class for crawling web pages
+with support for both single and multiple URL processing.
+"""
+
+import asyncio
+from typing import Dict, List, Optional, Tuple, Union, Callable
+
+from crawl4ai.models import CrawlResultContainer, CrawlResult
+from crawl4ai.pipeline.pipeline import create_pipeline
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+from crawl4ai.browser.browser_hub import BrowserHub
+
+# Type definitions
+UrlList = List[str]
+UrlBatch = Tuple[List[str], CrawlerRunConfig]
+UrlFullBatch = Tuple[List[str], BrowserConfig, CrawlerRunConfig]
+BatchType = Union[UrlList, UrlBatch, UrlFullBatch]
+ProgressCallback = Callable[[str, str, Optional[CrawlResultContainer]], None]
+RetryStrategy = Callable[[str, int, Exception], Tuple[bool, float]]
+
+class Crawler:
+    """High-level utility class for crawling web pages.
+    
+    This class provides simplified methods for crawling both single URLs
+    and batches of URLs, with parallel processing capabilities.
+    """
+    
+    @classmethod
+    async def crawl(
+        cls, 
+        urls: Union[str, List[str]],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        browser_hub: Optional[BrowserHub] = None,
+        logger: Optional[AsyncLogger] = None,
+        max_retries: int = 0,
+        retry_delay: float = 1.0,
+        use_new_loop: bool = True  # By default use a new loop for safety
+    ) -> Union[CrawlResultContainer, Dict[str, CrawlResultContainer]]:
+        """Crawl one or more URLs with the specified configurations.
+        
+        Args:
+            urls: Single URL or list of URLs to crawl
+            browser_config: Optional browser configuration
+            crawler_config: Optional crawler run configuration
+            browser_hub: Optional shared browser hub
+            logger: Optional logger instance
+            max_retries: Maximum number of retries for failed requests
+            retry_delay: Delay between retries in seconds
+            
+        Returns:
+            For a single URL: CrawlResultContainer with crawl results
+            For multiple URLs: Dict mapping URLs to their CrawlResultContainer results
+        """
+        # Handle single URL case
+        if isinstance(urls, str):
+            return await cls._crawl_single_url(
+                urls,
+                browser_config,
+                crawler_config,
+                browser_hub,
+                logger,
+                max_retries,
+                retry_delay,
+                use_new_loop
+            )
+        
+        # Handle multiple URLs case (sequential processing)
+        results = {}
+        for url in urls:
+            results[url] = await cls._crawl_single_url(
+                url,
+                browser_config,
+                crawler_config,
+                browser_hub,
+                logger,
+                max_retries,
+                retry_delay,
+                use_new_loop
+            )
+        
+        return results
+    
+    @classmethod
+    async def _crawl_single_url(
+        cls,
+        url: str,
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        browser_hub: Optional[BrowserHub] = None,
+        logger: Optional[AsyncLogger] = None,
+        max_retries: int = 0,
+        retry_delay: float = 1.0,
+        use_new_loop: bool = False
+    ) -> CrawlResultContainer:
+        """Internal method to crawl a single URL with retry logic."""
+        # Create a logger if none provided
+        if logger is None:
+            logger = AsyncLogger(verbose=True)
+            
+        # Create or use the provided crawler config
+        if crawler_config is None:
+            crawler_config = CrawlerRunConfig()
+            
+        attempts = 0
+        last_error = None
+        
+        # For testing purposes, each crawler gets a new event loop to avoid conflicts
+        # This is especially important in test suites where multiple tests run in sequence
+        if use_new_loop:
+            old_loop = asyncio.get_event_loop()
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            
+        while attempts <= max_retries:
+            try:
+                # Create a pipeline
+                pipeline_args = {}
+                if browser_config:
+                    pipeline_args["browser_config"] = browser_config
+                if browser_hub:
+                    pipeline_args["browser_hub"] = browser_hub
+                if logger:
+                    pipeline_args["logger"] = logger
+                    
+                pipeline = await create_pipeline(**pipeline_args)
+                
+                # Perform the crawl
+                result = await pipeline.crawl(url=url, config=crawler_config)
+                
+                # Close the pipeline if we created it (not using a shared hub)
+                if not browser_hub:
+                    await pipeline.close()
+                    
+                # Restore the original event loop if we created a new one
+                if use_new_loop:
+                    asyncio.set_event_loop(old_loop)
+                    loop.close()
+                    
+                return result
+                
+            except Exception as e:
+                last_error = e
+                attempts += 1
+                
+                if attempts <= max_retries:
+                    logger.warning(
+                        message="Crawl attempt {attempt} failed for {url}: {error}. Retrying in {delay}s...",
+                        tag="RETRY",
+                        params={
+                            "attempt": attempts,
+                            "url": url,
+                            "error": str(e),
+                            "delay": retry_delay
+                        }
+                    )
+                    await asyncio.sleep(retry_delay)
+                else:
+                    logger.error(
+                        message="All {attempts} crawl attempts failed for {url}: {error}",
+                        tag="FAILED",
+                        params={
+                            "attempts": attempts,
+                            "url": url,
+                            "error": str(e)
+                        }
+                    )
+        
+        # If we get here, all attempts failed
+        result = CrawlResultContainer(
+            CrawlResult(
+                url=url,
+                html="",
+                success=False,
+                error_message=f"All {attempts} crawl attempts failed: {str(last_error)}"
+            )
+        )
+        
+        # Restore the original event loop if we created a new one
+        if use_new_loop:
+            asyncio.set_event_loop(old_loop)
+            loop.close()
+            
+        return result
+    
+    @classmethod
+    async def parallel_crawl(
+        cls,
+        url_batches: Union[List[str], List[Union[UrlBatch, UrlFullBatch]]],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        browser_hub: Optional[BrowserHub] = None,
+        logger: Optional[AsyncLogger] = None,
+        concurrency: int = 5,
+        max_retries: int = 0,
+        retry_delay: float = 1.0,
+        retry_strategy: Optional[RetryStrategy] = None,
+        progress_callback: Optional[ProgressCallback] = None,
+        use_new_loop: bool = True  # By default use a new loop for safety
+    ) -> Dict[str, CrawlResultContainer]:
+        """Crawl multiple URLs in parallel with concurrency control.
+        
+        Args:
+            url_batches: List of URLs or list of URL batches with configurations
+            browser_config: Default browser configuration (used if not in batch)
+            crawler_config: Default crawler configuration (used if not in batch)
+            browser_hub: Optional shared browser hub for resource efficiency
+            logger: Optional logger instance
+            concurrency: Maximum number of concurrent crawls
+            max_retries: Maximum number of retries for failed requests
+            retry_delay: Delay between retries in seconds
+            retry_strategy: Optional custom retry strategy function
+            progress_callback: Optional callback for progress reporting
+            
+        Returns:
+            Dict mapping URLs to their CrawlResultContainer results
+        """
+        # Create a logger if none provided
+        if logger is None:
+            logger = AsyncLogger(verbose=True)
+        
+        # For testing purposes, each crawler gets a new event loop to avoid conflicts
+        # This is especially important in test suites where multiple tests run in sequence
+        if use_new_loop:
+            old_loop = asyncio.get_event_loop()
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            
+        # Process batches to consistent format
+        processed_batches = cls._process_url_batches(
+            url_batches, browser_config, crawler_config
+        )
+        
+        # Initialize results dictionary
+        results = {}
+        
+        # Create semaphore for concurrency control
+        semaphore = asyncio.Semaphore(concurrency)
+        
+        # Create shared browser hub if not provided
+        shared_hub = browser_hub
+        if not shared_hub:
+            shared_hub = await BrowserHub.get_browser_manager(
+                config=browser_config or BrowserConfig(),
+                logger=logger,
+                max_browsers_per_config=concurrency,
+                max_pages_per_browser=1,
+                initial_pool_size=min(concurrency, 3)  # Start with a reasonable number
+            )
+        
+        try:
+            # Create worker function for each URL
+            async def process_url(url, b_config, c_config):
+                async with semaphore:
+                    # Report start if callback provided
+                    if progress_callback:
+                        await progress_callback("started", url)
+                    
+                    attempts = 0
+                    last_error = None
+                    
+                    while attempts <= max_retries:
+                        try:
+                            # Create a pipeline using the shared hub
+                            pipeline = await create_pipeline(
+                                browser_config=b_config,
+                                browser_hub=shared_hub,
+                                logger=logger
+                            )
+                            
+                            # Perform the crawl
+                            result = await pipeline.crawl(url=url, config=c_config)
+                            
+                            # Report completion if callback provided
+                            if progress_callback:
+                                await progress_callback("completed", url, result)
+                                
+                            return url, result
+                            
+                        except Exception as e:
+                            last_error = e
+                            attempts += 1
+                            
+                            # Determine if we should retry and with what delay
+                            should_retry = attempts <= max_retries
+                            delay = retry_delay
+                            
+                            # Use custom retry strategy if provided
+                            if retry_strategy and should_retry:
+                                try:
+                                    should_retry, delay = await retry_strategy(url, attempts, e)
+                                except Exception as strategy_error:
+                                    logger.error(
+                                        message="Error in retry strategy: {error}",
+                                        tag="RETRY",
+                                        params={"error": str(strategy_error)}
+                                    )
+                            
+                            if should_retry:
+                                logger.warning(
+                                    message="Crawl attempt {attempt} failed for {url}: {error}. Retrying in {delay}s...",
+                                    tag="RETRY",
+                                    params={
+                                        "attempt": attempts,
+                                        "url": url,
+                                        "error": str(e),
+                                        "delay": delay
+                                    }
+                                )
+                                await asyncio.sleep(delay)
+                            else:
+                                logger.error(
+                                    message="All {attempts} crawl attempts failed for {url}: {error}",
+                                    tag="FAILED",
+                                    params={
+                                        "attempts": attempts,
+                                        "url": url,
+                                        "error": str(e)
+                                    }
+                                )
+                                break
+                    
+                    # If we get here, all attempts failed
+                    error_result = CrawlResultContainer(
+                        CrawlResult(
+                            url=url,
+                            html="",
+                            success=False,
+                            error_message=f"All {attempts} crawl attempts failed: {str(last_error)}"
+                        )
+                    )
+                    
+                    # Report completion with error if callback provided
+                    if progress_callback:
+                        await progress_callback("completed", url, error_result)
+                        
+                    return url, error_result
+            
+            # Create tasks for all URLs
+            tasks = []
+            for urls, b_config, c_config in processed_batches:
+                for url in urls:
+                    tasks.append(process_url(url, b_config, c_config))
+            
+            # Run all tasks and collect results
+            for completed_task in asyncio.as_completed(tasks):
+                url, result = await completed_task
+                results[url] = result
+                
+            return results
+            
+        finally:
+            # Clean up the hub only if we created it
+            if not browser_hub and shared_hub:
+                await shared_hub.close()
+                
+            # Restore the original event loop if we created a new one
+            if use_new_loop:
+                asyncio.set_event_loop(old_loop)
+                loop.close()
+    
+    @classmethod
+    def _process_url_batches(
+        cls,
+        url_batches: Union[List[str], List[Union[UrlBatch, UrlFullBatch]]],
+        default_browser_config: Optional[BrowserConfig],
+        default_crawler_config: Optional[CrawlerRunConfig]
+    ) -> List[Tuple[List[str], BrowserConfig, CrawlerRunConfig]]:
+        """Process URL batches into a consistent format.
+        
+        Converts various input formats into a consistent list of
+        (urls, browser_config, crawler_config) tuples.
+        """
+        processed_batches = []
+        
+        # Handle case where input is just a list of URLs
+        if all(isinstance(item, str) for item in url_batches):
+            urls = url_batches
+            browser_config = default_browser_config or BrowserConfig()
+            crawler_config = default_crawler_config or CrawlerRunConfig()
+            processed_batches.append((urls, browser_config, crawler_config))
+            return processed_batches
+        
+        # Process each batch
+        for batch in url_batches:
+            # Handle case: (urls, crawler_config)
+            if len(batch) == 2 and isinstance(batch[1], CrawlerRunConfig):
+                urls, c_config = batch
+                b_config = default_browser_config or BrowserConfig()
+                processed_batches.append((urls, b_config, c_config))
+                
+            # Handle case: (urls, browser_config, crawler_config)
+            elif len(batch) == 3 and isinstance(batch[1], BrowserConfig) and isinstance(batch[2], CrawlerRunConfig):
+                processed_batches.append(batch)
+                
+            # Fallback for unknown formats - assume it's just a list of URLs
+            else:
+                urls = batch
+                browser_config = default_browser_config or BrowserConfig()
+                crawler_config = default_crawler_config or CrawlerRunConfig()
+                processed_batches.append((urls, browser_config, crawler_config))
+                
+        return processed_batches

crawl4ai/pipeline/middlewares.py

@@ -0,0 +1,702 @@
+import time
+import sys
+from typing import Dict, Any, List
+import json
+
+from crawl4ai.models import (
+    CrawlResult,
+    MarkdownGenerationResult,
+    ScrapingResult,
+    CrawlResultContainer,
+)
+from crawl4ai.async_database import async_db_manager
+from crawl4ai.cache_context import CacheMode, CacheContext
+from crawl4ai.utils import (
+    sanitize_input_encode,
+    InvalidCSSSelectorError,
+    fast_format_html,
+    create_box_message,
+    get_error_context,
+)
+
+
+async def initialize_context_middleware(context: Dict[str, Any]) -> int:
+    """Initialize the context with basic configuration and validation"""
+    url = context.get("url")
+    config = context.get("config")
+    
+    if not isinstance(url, str) or not url:
+        context["error_message"] = "Invalid URL, make sure the URL is a non-empty string"
+        return 0
+    
+    # Default to ENABLED if no cache mode specified
+    if config.cache_mode is None:
+        config.cache_mode = CacheMode.ENABLED
+    
+    # Create cache context
+    context["cache_context"] = CacheContext(url, config.cache_mode, False)
+    context["start_time"] = time.perf_counter()
+    
+    return 1
+
+# middlewares.py additions
+
+async def browser_hub_middleware(context: Dict[str, Any]) -> int:
+    """
+    Initialize or connect to a Browser-Hub and add it to the pipeline context.
+    
+    This middleware handles browser hub initialization for all three scenarios:
+    1. Default configuration when nothing is specified
+    2. Custom configuration when browser_config is provided
+    3. Connection to existing hub when browser_hub_connection is provided
+    
+    Args:
+        context: The pipeline context dictionary
+        
+    Returns:
+        int: 1 for success, 0 for failure
+    """
+    from crawl4ai.browser.browser_hub import BrowserHub
+    
+    try:
+        # Get configuration from context
+        browser_config = context.get("browser_config")
+        browser_hub_id = context.get("browser_hub_id")
+        browser_hub_connection = context.get("browser_hub_connection")
+        logger = context.get("logger")
+        
+        # If we already have a browser hub in context, use it
+        if context.get("browser_hub"):
+            return 1
+        
+        # Get or create Browser-Hub
+        browser_hub = await BrowserHub.get_browser_manager(
+            config=browser_config,
+            hub_id=browser_hub_id,
+            connection_info=browser_hub_connection,
+            logger=logger
+        )
+        
+        # Add to context
+        context["browser_hub"] = browser_hub
+        return 1
+    except Exception as e:
+        context["error_message"] = f"Failed to initialize browser hub: {str(e)}"
+        return 0
+
+
+async def fetch_content_middleware(context: Dict[str, Any]) -> int:
+    """
+    Fetch content from the web using the browser hub.
+    
+    This middleware uses the browser hub to get pages for crawling,
+    and properly releases them back to the pool when done.
+    
+    Args:
+        context: The pipeline context dictionary
+        
+    Returns:
+        int: 1 for success, 0 for failure
+    """
+    url = context.get("url")
+    config = context.get("config")
+    browser_hub = context.get("browser_hub")
+    logger = context.get("logger")
+    
+    # Skip if using cached result
+    if context.get("cached_result") and context.get("html"):
+        return 1
+    
+    try:
+        # Create crawler strategy without initializing its browser manager
+        from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
+        
+        crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+            browser_config=browser_hub.config if browser_hub else None,
+            logger=logger
+        )
+        
+        # Replace the browser manager with our shared instance
+        crawler_strategy.browser_manager = browser_hub
+        
+        # Perform crawl without trying to initialize the browser
+        # The crawler will use the provided browser_manager to get pages
+        async_response = await crawler_strategy.crawl(url, config=config)
+        
+        # Store results in context
+        context["html"] = async_response.html
+        context["screenshot_data"] = async_response.screenshot
+        context["pdf_data"] = async_response.pdf_data
+        context["js_execution_result"] = async_response.js_execution_result
+        context["async_response"] = async_response
+        
+        return 1
+    except Exception as e:
+        context["error_message"] = f"Error fetching content: {str(e)}"
+        return 0
+
+
+async def check_cache_middleware(context: Dict[str, Any]) -> int:
+    """Check if there's a cached result and load it if available"""
+    url = context.get("url")
+    config = context.get("config")
+    cache_context = context.get("cache_context")
+    logger = context.get("logger")
+    
+    # Initialize variables
+    context["cached_result"] = None
+    context["html"] = None
+    context["extracted_content"] = None
+    context["screenshot_data"] = None
+    context["pdf_data"] = None
+    
+    # Try to get cached result if appropriate
+    if cache_context.should_read():
+        cached_result = await async_db_manager.aget_cached_url(url)
+        context["cached_result"] = cached_result
+        
+        if cached_result:
+            html = sanitize_input_encode(cached_result.html)
+            extracted_content = sanitize_input_encode(cached_result.extracted_content or "")
+            extracted_content = None if not extracted_content or extracted_content == "[]" else extracted_content
+            
+            # If screenshot is requested but its not in cache, then set cache_result to None
+            screenshot_data = cached_result.screenshot
+            pdf_data = cached_result.pdf
+            
+            if config.screenshot and not screenshot_data:
+                context["cached_result"] = None
+            
+            if config.pdf and not pdf_data:
+                context["cached_result"] = None
+            
+            context["html"] = html
+            context["extracted_content"] = extracted_content
+            context["screenshot_data"] = screenshot_data
+            context["pdf_data"] = pdf_data
+            
+            logger.url_status(
+                url=cache_context.display_url,
+                success=bool(html),
+                timing=time.perf_counter() - context["start_time"],
+                tag="FETCH",
+            )
+    
+    return 1
+
+
+async def configure_proxy_middleware(context: Dict[str, Any]) -> int:
+    """Configure proxy if a proxy rotation strategy is available"""
+    config = context.get("config")
+    logger = context.get("logger")
+    
+    # Skip if using cached result
+    if context.get("cached_result") and context.get("html"):
+        return 1
+    
+    # Update proxy configuration from rotation strategy if available
+    if config and config.proxy_rotation_strategy:
+        next_proxy = await config.proxy_rotation_strategy.get_next_proxy()
+        if next_proxy:
+            logger.info(
+                message="Switch proxy: {proxy}",
+                tag="PROXY",
+                params={"proxy": next_proxy.server},
+            )
+            config.proxy_config = next_proxy
+    
+    return 1
+
+
+async def check_robots_txt_middleware(context: Dict[str, Any]) -> int:
+    """Check if the URL is allowed by robots.txt if enabled"""
+    url = context.get("url")
+    config = context.get("config")
+    browser_config = context.get("browser_config")
+    robots_parser = context.get("robots_parser")
+    
+    # Skip if using cached result
+    if context.get("cached_result") and context.get("html"):
+        return 1
+    
+    # Check robots.txt if enabled
+    if config and config.check_robots_txt:
+        if not await robots_parser.can_fetch(url, browser_config.user_agent):
+            context["crawl_result"] = CrawlResult(
+                url=url,
+                html="",
+                success=False,
+                status_code=403,
+                error_message="Access denied by robots.txt",
+                response_headers={"X-Robots-Status": "Blocked by robots.txt"}
+            )
+            return 0
+    
+    return 1
+
+
+async def fetch_content_middleware_(context: Dict[str, Any]) -> int:
+    """Fetch content from the web using the crawler strategy"""
+    url = context.get("url")
+    config = context.get("config")
+    crawler_strategy = context.get("crawler_strategy")
+    logger = context.get("logger")
+    
+    # Skip if using cached result
+    if context.get("cached_result") and context.get("html"):
+        return 1
+    
+    try:
+        t1 = time.perf_counter()
+        
+        if config.user_agent:
+            crawler_strategy.update_user_agent(config.user_agent)
+        
+        # Call CrawlerStrategy.crawl
+        async_response = await crawler_strategy.crawl(url, config=config)
+        
+        html = sanitize_input_encode(async_response.html)
+        screenshot_data = async_response.screenshot
+        pdf_data = async_response.pdf_data
+        js_execution_result = async_response.js_execution_result
+        
+        t2 = time.perf_counter()
+        logger.url_status(
+            url=context["cache_context"].display_url,
+            success=bool(html),
+            timing=t2 - t1,
+            tag="FETCH",
+        )
+        
+        context["html"] = html
+        context["screenshot_data"] = screenshot_data
+        context["pdf_data"] = pdf_data
+        context["js_execution_result"] = js_execution_result
+        context["async_response"] = async_response
+        
+        return 1
+    except Exception as e:
+        context["error_message"] = f"Error fetching content: {str(e)}"
+        return 0
+
+
+async def scrape_content_middleware(context: Dict[str, Any]) -> int:
+    """Apply scraping strategy to extract content"""
+    url = context.get("url")
+    html = context.get("html")
+    config = context.get("config")
+    extracted_content = context.get("extracted_content")
+    logger = context.get("logger")
+    
+    # Skip if already have a crawl result
+    if context.get("crawl_result"):
+        return 1
+    
+    try:
+        _url = url if not context.get("is_raw_html", False) else "Raw HTML"
+        t1 = time.perf_counter()
+        
+        # Get scraping strategy and ensure it has a logger
+        scraping_strategy = config.scraping_strategy
+        if not scraping_strategy.logger:
+            scraping_strategy.logger = logger
+        
+        # Process HTML content
+        params = config.__dict__.copy()
+        params.pop("url", None)
+        # Add keys from kwargs to params that don't exist in params
+        kwargs = context.get("kwargs", {})
+        params.update({k: v for k, v in kwargs.items() if k not in params.keys()})
+        
+        # Scraping Strategy Execution
+        result: ScrapingResult = scraping_strategy.scrap(url, html, **params)
+        
+        if result is None:
+            raise ValueError(f"Process HTML, Failed to extract content from the website: {url}")
+        
+        # Extract results - handle both dict and ScrapingResult
+        if isinstance(result, dict):
+            cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
+            media = result.get("media", {})
+            links = result.get("links", {})
+            metadata = result.get("metadata", {})
+        else:
+            cleaned_html = sanitize_input_encode(result.cleaned_html)
+            media = result.media.model_dump()
+            links = result.links.model_dump()
+            metadata = result.metadata
+        
+        context["cleaned_html"] = cleaned_html
+        context["media"] = media
+        context["links"] = links
+        context["metadata"] = metadata
+        
+        # Log processing completion
+        logger.info(
+            message="{url:.50}... | Time: {timing}s",
+            tag="SCRAPE",
+            params={
+                "url": _url,
+                "timing": int((time.perf_counter() - t1) * 1000) / 1000,
+            },
+        )
+        
+        return 1
+    except InvalidCSSSelectorError as e:
+        context["error_message"] = str(e)
+        return 0
+    except Exception as e:
+        context["error_message"] = f"Process HTML, Failed to extract content from the website: {url}, error: {str(e)}"
+        return 0
+
+
+async def generate_markdown_middleware(context: Dict[str, Any]) -> int:
+    """Generate markdown from cleaned HTML"""
+    url = context.get("url")
+    cleaned_html = context.get("cleaned_html")
+    config = context.get("config")
+    
+    # Skip if already have a crawl result
+    if context.get("crawl_result"):
+        return 1
+    
+    # Generate Markdown
+    markdown_generator = config.markdown_generator
+    
+    markdown_result: MarkdownGenerationResult = markdown_generator.generate_markdown(
+        cleaned_html=cleaned_html,
+        base_url=url,
+    )
+    
+    context["markdown_result"] = markdown_result
+    
+    return 1
+
+
+async def extract_structured_content_middleware(context: Dict[str, Any]) -> int:
+    """Extract structured content using extraction strategy"""
+    url = context.get("url")
+    extracted_content = context.get("extracted_content")
+    config = context.get("config")
+    markdown_result = context.get("markdown_result")
+    cleaned_html = context.get("cleaned_html")
+    logger = context.get("logger")
+    
+    # Skip if already have a crawl result or extracted content
+    if context.get("crawl_result") or bool(extracted_content):
+        return 1
+    
+    from crawl4ai.chunking_strategy import IdentityChunking
+    from crawl4ai.extraction_strategy import NoExtractionStrategy
+    
+    if config.extraction_strategy and not isinstance(config.extraction_strategy, NoExtractionStrategy):
+        t1 = time.perf_counter()
+        _url = url if not context.get("is_raw_html", False) else "Raw HTML"
+        
+        # Choose content based on input_format
+        content_format = config.extraction_strategy.input_format
+        if content_format == "fit_markdown" and not markdown_result.fit_markdown:
+            logger.warning(
+                message="Fit markdown requested but not available. Falling back to raw markdown.",
+                tag="EXTRACT",
+                params={"url": _url},
+            )
+            content_format = "markdown"
+        
+        content = {
+            "markdown": markdown_result.raw_markdown,
+            "html": context.get("html"),
+            "cleaned_html": cleaned_html,
+            "fit_markdown": markdown_result.fit_markdown,
+        }.get(content_format, markdown_result.raw_markdown)
+        
+        # Use IdentityChunking for HTML input, otherwise use provided chunking strategy
+        chunking = (
+            IdentityChunking()
+            if content_format in ["html", "cleaned_html"]
+            else config.chunking_strategy
+        )
+        sections = chunking.chunk(content)
+        extracted_content = config.extraction_strategy.run(url, sections)
+        extracted_content = json.dumps(
+            extracted_content, indent=4, default=str, ensure_ascii=False
+        )
+        
+        context["extracted_content"] = extracted_content
+        
+        # Log extraction completion
+        logger.info(
+            message="Completed for {url:.50}... | Time: {timing}s",
+            tag="EXTRACT",
+            params={"url": _url, "timing": time.perf_counter() - t1},
+        )
+    
+    return 1
+
+
+async def format_html_middleware(context: Dict[str, Any]) -> int:
+    """Format HTML if prettify is enabled"""
+    config = context.get("config")
+    cleaned_html = context.get("cleaned_html")
+    
+    # Skip if already have a crawl result
+    if context.get("crawl_result"):
+        return 1
+    
+    # Apply HTML formatting if requested
+    if config.prettiify and cleaned_html:
+        context["cleaned_html"] = fast_format_html(cleaned_html)
+    
+    return 1
+
+
+async def write_cache_middleware(context: Dict[str, Any]) -> int:
+    """Write result to cache if appropriate"""
+    cache_context = context.get("cache_context")
+    cached_result = context.get("cached_result")
+    
+    # Skip if already have a crawl result or not using cache
+    if context.get("crawl_result") or not cache_context.should_write() or bool(cached_result):
+        return 1
+    
+    # We'll create the CrawlResult in build_result_middleware and cache it there
+    # to avoid creating it twice
+    
+    return 1
+
+
+async def build_result_middleware(context: Dict[str, Any]) -> int:
+    """Build the final CrawlResult object"""
+    url = context.get("url")
+    html = context.get("html", "")
+    cache_context = context.get("cache_context")
+    cached_result = context.get("cached_result")
+    config = context.get("config")
+    logger = context.get("logger")
+    
+    # If we already have a crawl result (from an earlier middleware like robots.txt check)
+    if context.get("crawl_result"):
+        result = context["crawl_result"]
+        context["final_result"] = CrawlResultContainer(result)
+        return 1
+    
+    # If we have a cached result
+    if cached_result and html:
+        logger.success(
+            message="{url:.50}... | Status: {status} | Total: {timing}",
+            tag="COMPLETE",
+            params={
+                "url": cache_context.display_url,
+                "status": True,
+                "timing": f"{time.perf_counter() - context['start_time']:.2f}s",
+            },
+            colors={"status": "green", "timing": "yellow"},
+        )
+        
+        cached_result.success = bool(html)
+        cached_result.session_id = getattr(config, "session_id", None)
+        cached_result.redirected_url = cached_result.redirected_url or url
+        context["final_result"] = CrawlResultContainer(cached_result)
+        return 1
+    
+    # Build a new result
+    try:
+        # Get all necessary components from context
+        cleaned_html = context.get("cleaned_html", "")
+        markdown_result = context.get("markdown_result")
+        media = context.get("media", {})
+        links = context.get("links", {})
+        metadata = context.get("metadata", {})
+        screenshot_data = context.get("screenshot_data")
+        pdf_data = context.get("pdf_data")
+        extracted_content = context.get("extracted_content")
+        async_response = context.get("async_response")
+        
+        # Create the CrawlResult
+        crawl_result = CrawlResult(
+            url=url,
+            html=html,
+            cleaned_html=cleaned_html,
+            markdown=markdown_result,
+            media=media,
+            links=links,
+            metadata=metadata,
+            screenshot=screenshot_data,
+            pdf=pdf_data,
+            extracted_content=extracted_content,
+            success=bool(html),
+            error_message="",
+        )
+        
+        # Add response details if available
+        if async_response:
+            crawl_result.status_code = async_response.status_code
+            crawl_result.redirected_url = async_response.redirected_url or url
+            crawl_result.response_headers = async_response.response_headers
+            crawl_result.downloaded_files = async_response.downloaded_files
+            crawl_result.js_execution_result = context.get("js_execution_result")
+            crawl_result.ssl_certificate = async_response.ssl_certificate
+        
+        crawl_result.session_id = getattr(config, "session_id", None)
+        
+        # Log completion
+        logger.success(
+            message="{url:.50}... | Status: {status} | Total: {timing}",
+            tag="COMPLETE",
+            params={
+                "url": cache_context.display_url,
+                "status": crawl_result.success,
+                "timing": f"{time.perf_counter() - context['start_time']:.2f}s",
+            },
+            colors={
+                "status": "green" if crawl_result.success else "red",
+                "timing": "yellow",
+            },
+        )
+        
+        # Update cache if appropriate
+        if cache_context.should_write() and not bool(cached_result):
+            await async_db_manager.acache_url(crawl_result)
+        
+        context["final_result"] = CrawlResultContainer(crawl_result)
+        return 1
+    except Exception as e:
+        error_context = get_error_context(sys.exc_info())
+        
+        error_message = (
+            f"Unexpected error in build_result at line {error_context['line_no']} "
+            f"in {error_context['function']} ({error_context['filename']}):\n"
+            f"Error: {str(e)}\n\n"
+            f"Code context:\n{error_context['code_context']}"
+        )
+        
+        logger.error_status(
+            url=url,
+            error=create_box_message(error_message, type="error"),
+            tag="ERROR",
+        )
+        
+        context["final_result"] = CrawlResultContainer(
+            CrawlResult(
+                url=url, html="", success=False, error_message=error_message
+            )
+        )
+        return 1
+
+
+async def handle_error_middleware(context: Dict[str, Any]) -> Dict[str, Any]:
+    """Error handler middleware"""
+    url = context.get("url", "")
+    error_message = context.get("error_message", "Unknown error")
+    logger = context.get("logger")
+    
+    # Log the error
+    if logger:
+        logger.error_status(
+            url=url,
+            error=create_box_message(error_message, type="error"),
+            tag="ERROR",
+        )
+    
+    # Create a failure result
+    context["final_result"] = CrawlResultContainer(
+        CrawlResult(
+            url=url, html="", success=False, error_message=error_message
+        )
+    )
+    
+    return context
+
+
+# Custom middlewares as requested
+
+async def sentiment_analysis_middleware(context: Dict[str, Any]) -> int:
+    """Analyze sentiment of generated markdown using TextBlob"""
+    from textblob import TextBlob
+    
+    markdown_result = context.get("markdown_result")
+    
+    # Skip if no markdown or already failed
+    if not markdown_result or not context.get("success", True):
+        return 1
+    
+    try:
+        # Get raw markdown text
+        raw_markdown = markdown_result.raw_markdown
+        
+        # Analyze sentiment
+        blob = TextBlob(raw_markdown)
+        sentiment = blob.sentiment
+        
+        # Add sentiment to context
+        context["sentiment_analysis"] = {
+            "polarity": sentiment.polarity,  # -1.0 to 1.0 (negative to positive)
+            "subjectivity": sentiment.subjectivity,  # 0.0 to 1.0 (objective to subjective)
+            "classification": "positive" if sentiment.polarity > 0.1 else 
+                             "negative" if sentiment.polarity < -0.1 else "neutral"
+        }
+        
+        return 1
+    except Exception as e:
+        # Don't fail the pipeline on sentiment analysis failure
+        context["sentiment_analysis_error"] = str(e)
+        return 1
+
+
+async def log_timing_middleware(context: Dict[str, Any], name: str) -> int:
+    """Log timing information for a specific point in the pipeline"""
+    context[f"_timing_mark_{name}"] = time.perf_counter()
+    
+    # Calculate duration if we have a start time
+    start_key = f"_timing_start_{name}"
+    if start_key in context:
+        duration = context[f"_timing_mark_{name}"] - context[start_key]
+        context[f"_timing_duration_{name}"] = duration
+        
+        # Log the timing if we have a logger
+        logger = context.get("logger")
+        if logger:
+            logger.info(
+                message="{name} completed in {duration:.2f}s",
+                tag="TIMING",
+                params={"name": name, "duration": duration},
+            )
+    
+    return 1
+
+
+async def validate_url_middleware(context: Dict[str, Any], patterns: List[str]) -> int:
+    """Validate URL against glob patterns"""
+    import fnmatch
+    url = context.get("url", "")
+    
+    # If no patterns provided, allow all
+    if not patterns:
+        return 1
+    
+    # Check if URL matches any of the allowed patterns
+    for pattern in patterns:
+        if fnmatch.fnmatch(url, pattern):
+            return 1
+    
+    # If we get here, URL didn't match any patterns
+    context["error_message"] = f"URL '{url}' does not match any allowed patterns"
+    return 0
+
+
+# Update the default middleware list function
+def create_default_middleware_list():
+    """Return the default list of middleware functions for the pipeline."""
+    return [
+        initialize_context_middleware,
+        check_cache_middleware,
+        browser_hub_middleware,  # Add browser hub middleware before fetch_content
+        configure_proxy_middleware,
+        check_robots_txt_middleware,
+        fetch_content_middleware,
+        scrape_content_middleware,
+        generate_markdown_middleware,
+        extract_structured_content_middleware,
+        format_html_middleware,
+        build_result_middleware
+    ]

crawl4ai/pipeline/pipeline.py

@@ -0,0 +1,297 @@
+
+import time
+import asyncio
+from typing import Callable, Dict, List, Any, Optional, Awaitable, Union, TypedDict, Tuple, Coroutine
+
+from .middlewares import create_default_middleware_list, handle_error_middleware
+from crawl4ai.models import CrawlResultContainer, CrawlResult
+from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy, AsyncPlaywrightCrawlerStrategy
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_logger import AsyncLogger
+
+
+class CrawlSpec(TypedDict, total=False):
+    """Specification for a single crawl operation in batch_crawl."""
+    url: str
+    config: Optional[CrawlerRunConfig]
+    browser_config: Optional[BrowserConfig]
+
+class BatchStatus(TypedDict, total=False):
+    """Status information for batch crawl operations."""
+    total: int
+    processed: int
+    succeeded: int
+    failed: int
+    in_progress: int
+    duration: float
+    
+class Pipeline:
+    """
+    A pipeline processor that executes a series of async middleware functions.
+    Each middleware function receives a context dictionary, updates it,
+    and returns 1 for success or 0 for failure.
+    """
+
+    def __init__(
+        self, 
+        middleware: List[Callable[[Dict[str, Any]], Awaitable[int]]] = None,
+        error_handler: Optional[Callable[[Dict[str, Any]], Awaitable[Dict[str, Any]]]] = None,
+        after_middleware_callback: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None,
+        crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
+        browser_config: Optional[BrowserConfig] = None,
+        logger: Optional[AsyncLogger] = None,
+        _initial_context: Optional[Dict[str, Any]] = None
+    ):
+        self.middleware = middleware or create_default_middleware_list()
+        self.error_handler = error_handler or handle_error_middleware
+        self.after_middleware_callback = after_middleware_callback
+        self.browser_config = browser_config or BrowserConfig()
+        self.logger = logger or AsyncLogger(verbose=self.browser_config.verbose)
+        self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy(
+            browser_config=self.browser_config,
+            logger=self.logger
+        )
+        self._initial_context = _initial_context
+        self._strategy_initialized = False
+    
+    async def _initialize_strategy__(self):
+        """Initialize the crawler strategy if not already initialized"""
+        if not self.crawler_strategy:
+            self.crawler_strategy = AsyncPlaywrightCrawlerStrategy(
+                browser_config=self.browser_config,
+                logger=self.logger
+            )
+        
+        if not self._strategy_initialized:
+            await self.crawler_strategy.__aenter__()
+            self._strategy_initialized = True
+    
+    async def _initialize_strategy(self):
+        """Initialize the crawler strategy if not already initialized"""
+        # With our new approach, we don't need to create the crawler strategy here
+        # as it will be created on-demand in fetch_content_middleware
+        
+        # Just ensure browser hub is available if needed
+        if hasattr(self, "_initial_context") and "browser_hub" not in self._initial_context:
+            # If a browser_config was provided but no browser_hub yet, 
+            # we'll let the browser_hub_middleware handle creating it
+            pass
+        
+        # Mark as initialized to prevent repeated initialization attempts
+        self._strategy_initialized = True
+
+    async def start(self):
+        """Start the crawler strategy and prepare it for use"""
+        if not self._strategy_initialized:
+            await self._initialize_strategy()
+            self._strategy_initialized = True
+        if self.crawler_strategy:
+            await self.crawler_strategy.__aenter__()
+            self._strategy_initialized = True
+        else:
+            raise ValueError("Crawler strategy is not initialized.")
+    
+    async def close(self):
+        """Close the crawler strategy and clean up resources"""
+        await self.stop()
+
+    async def stop(self):
+        """Close the crawler strategy and clean up resources"""
+        if self._strategy_initialized and self.crawler_strategy:
+            await self.crawler_strategy.__aexit__(None, None, None)
+            self._strategy_initialized = False
+    
+    async def __aenter__(self):
+        await self.start()
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+    
+    async def crawl(self, url: str, config: Optional[CrawlerRunConfig] = None, **kwargs) -> CrawlResultContainer:
+        """
+        Crawl a URL and process it through the pipeline.
+        
+        Args:
+            url: The URL to crawl
+            config: Optional configuration for the crawl
+            **kwargs: Additional arguments to pass to the middleware
+            
+        Returns:
+            CrawlResultContainer: The result of the crawl
+        """
+        # Initialize strategy if needed
+        await self._initialize_strategy()
+        
+        # Create the initial context
+        context = {
+            "url": url,
+            "config": config or CrawlerRunConfig(),
+            "browser_config": self.browser_config,
+            "logger": self.logger,
+            "crawler_strategy": self.crawler_strategy,
+            "kwargs": kwargs
+        }
+        
+        # Process the pipeline
+        result_context = await self.process(context)
+        
+        # Return the final result
+        return result_context.get("final_result")
+    
+    async def process(self, initial_context: Dict[str, Any] = None) -> Dict[str, Any]:
+        """
+        Process all middleware functions with the given context.
+        
+        Args:
+            initial_context: Initial context dictionary, defaults to empty dict
+            
+        Returns:
+            Updated context dictionary after all middleware have been processed
+        """
+        context = {**self._initial_context}
+        if initial_context:
+            context.update(initial_context)
+        
+        # Record pipeline start time
+        context["_pipeline_start_time"] = time.perf_counter()
+        
+        for middleware_fn in self.middleware:
+            # Get middleware name for logging
+            middleware_name = getattr(middleware_fn, '__name__', str(middleware_fn))
+            
+            # Record start time for this middleware
+            start_time = time.perf_counter()
+            context[f"_timing_start_{middleware_name}"] = start_time
+            
+            try:
+                # Execute middleware (all middleware functions are async)
+                result = await middleware_fn(context)
+                
+                # Record completion time
+                end_time = time.perf_counter()
+                context[f"_timing_end_{middleware_name}"] = end_time
+                context[f"_timing_duration_{middleware_name}"] = end_time - start_time
+                
+                # Execute after-middleware callback if provided
+                if self.after_middleware_callback:
+                    await self.after_middleware_callback(middleware_name, context)
+                
+                # Convert boolean returns to int (True->1, False->0)
+                if isinstance(result, bool):
+                    result = 1 if result else 0
+                
+                # Handle failure
+                if result == 0:
+                    if self.error_handler:
+                        context["_error_in"] = middleware_name
+                        context["_error_at"] = time.perf_counter()
+                        return await self._handle_error(context)
+                    else:
+                        context["success"] = False
+                        context["error_message"] = f"Pipeline failed at {middleware_name}"
+                        break
+            except Exception as e:
+                # Record error information
+                context["_error_in"] = middleware_name
+                context["_error_at"] = time.perf_counter()
+                context["_exception"] = e
+                context["success"] = False
+                context["error_message"] = f"Exception in {middleware_name}: {str(e)}"
+                
+                # Call error handler if available
+                if self.error_handler:
+                    return await self._handle_error(context)
+                break
+        
+        # Record pipeline completion time
+        pipeline_end_time = time.perf_counter()
+        context["_pipeline_end_time"] = pipeline_end_time
+        context["_pipeline_duration"] = pipeline_end_time - context["_pipeline_start_time"]
+        
+        # Set success to True if not already set (no failures)
+        if "success" not in context:
+            context["success"] = True
+        
+        return context
+    
+    async def _handle_error(self, context: Dict[str, Any]) -> Dict[str, Any]:
+        """Handle errors by calling the error handler"""
+        try:
+            return await self.error_handler(context)
+        except Exception as e:
+            # If error handler fails, update context with this new error
+            context["_error_handler_exception"] = e
+            context["error_message"] = f"Error handler failed: {str(e)}"
+            return context
+
+
+
+async def create_pipeline(
+    middleware_list=None, 
+    error_handler=None,
+    after_middleware_callback=None,
+    browser_config=None,
+    browser_hub_id=None,
+    browser_hub_connection=None,
+    browser_hub=None,
+    logger=None
+) -> Pipeline:
+    """
+    Factory function to create a pipeline with Browser-Hub integration.
+    
+    Args:
+        middleware_list: List of middleware functions
+        error_handler: Error handler middleware
+        after_middleware_callback: Callback after middleware execution
+        browser_config: Configuration for the browser
+        browser_hub_id: ID for browser hub instance
+        browser_hub_connection: Connection string for existing browser hub
+        browser_hub: Existing browser hub instance to use
+        logger: Logger instance
+        
+    Returns:
+        Pipeline: Configured pipeline instance
+    """
+    # Use default middleware list if none provided
+    middleware = middleware_list or create_default_middleware_list()
+    
+    # Create the pipeline
+    pipeline = Pipeline(
+        middleware=middleware,
+        error_handler=error_handler,
+        after_middleware_callback=after_middleware_callback,
+        logger=logger
+    )
+    
+    # Set browser-related attributes in the initial context
+    pipeline._initial_context = {
+        "browser_config": browser_config,
+        "browser_hub_id": browser_hub_id,
+        "browser_hub_connection": browser_hub_connection,
+        "browser_hub": browser_hub,
+        "logger": logger
+    }
+    
+    return pipeline
+
+
+
+
+# async def create_pipeline(
+#     middleware_list: Optional[List[Callable[[Dict[str, Any]], Awaitable[int]]]] = None, 
+#     error_handler: Optional[Callable[[Dict[str, Any]], Awaitable[Dict[str, Any]]]] = None,
+#     after_middleware_callback: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None,
+#     crawler_strategy = None,
+#     browser_config = None,
+#     logger = None
+# ) -> Pipeline:
+#     """Factory function to create a pipeline with the given middleware"""
+#     return Pipeline(
+#         middleware=middleware_list,
+#         error_handler=error_handler,
+#         after_middleware_callback=after_middleware_callback,
+#         crawler_strategy=crawler_strategy,
+#         browser_config=browser_config,
+#         logger=logger
+#     )

crawl4ai/pipeline/test_pipeline.py

@@ -0,0 +1,109 @@
+import asyncio
+from crawl4ai import (
+    BrowserConfig,
+    CrawlerRunConfig,
+    CacheMode,
+    DefaultMarkdownGenerator,
+    PruningContentFilter
+)
+from pipeline import Pipeline
+
+async def main():
+    # Create configuration objects
+    browser_config = BrowserConfig(headless=True, verbose=True)
+    crawler_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(
+                threshold=0.48, 
+                threshold_type="fixed", 
+                min_word_threshold=0
+            )
+        ),
+    )
+    
+    # Create and use pipeline with context manager
+    async with Pipeline(browser_config=browser_config) as pipeline:
+        result = await pipeline.crawl(
+            url="https://www.example.com", 
+            config=crawler_config
+        )
+        
+        # Print the result
+        print(f"URL: {result.url}")
+        print(f"Success: {result.success}")
+        
+        if result.success:
+            print("\nMarkdown excerpt:")
+            print(result.markdown.raw_markdown[:500] + "...")
+        else:
+            print(f"Error: {result.error_message}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+
+
+class CrawlTarget:
+    def __init__(self, urls, config=None):
+        self.urls = urls
+        self.config = config
+
+    def __repr__(self):
+        return f"CrawlTarget(urls={self.urls}, config={self.config})"
+    
+
+
+
+# async def main():
+#     # Create configuration objects
+#     browser_config = BrowserConfig(headless=True, verbose=True)
+    
+#     # Define different configurations
+#     config1 = CrawlerRunConfig(
+#         cache_mode=CacheMode.BYPASS,
+#         markdown_generator=DefaultMarkdownGenerator(
+#             content_filter=PruningContentFilter(threshold=0.48)
+#         ),
+#     )
+    
+#     config2 = CrawlerRunConfig(
+#         cache_mode=CacheMode.ENABLED,
+#         screenshot=True,
+#         pdf=True
+#     )
+    
+#     # Create crawl targets
+#     targets = [
+#         CrawlTarget(
+#             urls=["https://www.example.com", "https://www.wikipedia.org"],
+#             config=config1
+#         ),
+#         CrawlTarget(
+#             urls="https://news.ycombinator.com",  
+#             config=config2
+#         ),
+#         CrawlTarget(
+#             urls=["https://github.com", "https://stackoverflow.com", "https://python.org"],
+#             config=None
+#         )
+#     ]
+    
+#     # Create and use pipeline with context manager
+#     async with Pipeline(browser_config=browser_config) as pipeline:
+#         all_results = await pipeline.crawl_batch(targets)
+        
+#         for target_key, results in all_results.items():
+#             print(f"\n===== Results for {target_key} =====")
+#             print(f"Number of URLs crawled: {len(results)}")
+            
+#             for i, result in enumerate(results):
+#                 print(f"\nURL {i+1}: {result.url}")
+#                 print(f"Success: {result.success}")
+                
+#                 if result.success:
+#                     print(f"Content length: {len(result.markdown.raw_markdown)} chars")
+#                 else:
+#                     print(f"Error: {result.error_message}")
+
+# if __name__ == "__main__":
+#     asyncio.run(main())

crawl4ai/processors/pdf/__init__.py

@@ -0,0 +1,165 @@
+from pathlib import Path
+import asyncio
+from dataclasses import asdict
+from crawl4ai.async_logger import AsyncLogger
+from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy
+from crawl4ai.models import AsyncCrawlResponse, ScrapingResult 
+from crawl4ai.content_scraping_strategy import ContentScrapingStrategy
+from .processor import NaivePDFProcessorStrategy  # Assuming your current PDF code is in pdf_processor.py
+
+class PDFCrawlerStrategy(AsyncCrawlerStrategy):
+    def __init__(self, logger: AsyncLogger = None):
+        self.logger = logger
+        
+    async def crawl(self, url: str, **kwargs) -> AsyncCrawlResponse:
+        # Just pass through with empty HTML - scraper will handle actual processing
+        return AsyncCrawlResponse(
+            html="",  # Scraper will handle the real work
+            response_headers={"Content-Type": "application/pdf"},
+            status_code=200
+        )
+    
+    async def close(self):
+        pass        
+        
+    async def __aenter__(self):        
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+
+class PDFContentScrapingStrategy(ContentScrapingStrategy):
+    """
+    A content scraping strategy for PDF files.
+    
+    Attributes:
+        save_images_locally (bool): Whether to save images locally.
+        extract_images (bool): Whether to extract images from PDF.
+        image_save_dir (str): Directory to save extracted images.
+        logger (AsyncLogger): Logger instance for recording events and errors.
+        
+    Methods:
+        scrap(url: str, html: str, **params) -> ScrapingResult:
+            Scrap content from a PDF file.
+        ascrap(url: str, html: str, **kwargs) -> ScrapingResult:
+            Asynchronous version of scrap.
+            
+    Usage:
+        strategy = PDFContentScrapingStrategy(
+            save_images_locally=False,
+            extract_images=False,
+            image_save_dir=None,
+            logger=logger
+        )
+        
+    """
+    def __init__(self, 
+                 save_images_locally : bool = False,
+                 extract_images : bool = False,
+                 image_save_dir : str = None,
+                 batch_size: int = 4,
+                 logger: AsyncLogger = None):
+        self.logger = logger
+        self.pdf_processor = NaivePDFProcessorStrategy(
+            save_images_locally=save_images_locally,
+            extract_images=extract_images,
+            image_save_dir=image_save_dir,
+            batch_size=batch_size
+        )
+
+    def scrap(self, url: str, html: str, **params) -> ScrapingResult:
+        """
+        Scrap content from a PDF file.
+        
+        Args:
+            url (str): The URL of the PDF file.
+            html (str): The HTML content of the page.
+            **params: Additional parameters.
+        
+        Returns:
+            ScrapingResult: The scraped content.
+        """
+        # Download if URL or use local path
+        pdf_path = self._get_pdf_path(url)
+        try:
+            # Process PDF
+            # result = self.pdf_processor.process(Path(pdf_path))
+            result = self.pdf_processor.process_batch(Path(pdf_path))
+            
+            # Combine page HTML
+            cleaned_html = f"""
+        <html>
+            <head><meta name="pdf-pages" content="{len(result.pages)}"></head>
+            <body>
+                {''.join(f'<div class="pdf-page" data-page="{i+1}">{page.html}</div>'
+                         for i, page in enumerate(result.pages))}
+            </body>
+        </html>
+        """
+            
+            # Accumulate media and links with page numbers
+            media = {"images": []}
+            links = {"urls": []}
+            
+            for page in result.pages:
+                # Add page number to each image
+                for img in page.images:
+                    img["page"] = page.page_number
+                    media["images"].append(img)
+                
+                # Add page number to each link
+                for link in page.links:
+                    links["urls"].append({
+                        "url": link,
+                        "page": page.page_number
+                    })
+
+            return ScrapingResult(
+                cleaned_html=cleaned_html,
+                success=True,
+                media=media,
+                links=links,
+                metadata=asdict(result.metadata)
+            )
+        finally:
+            # Cleanup temp file if downloaded
+            if url.startswith(("http://", "https://")):
+                Path(pdf_path).unlink(missing_ok=True)
+
+    async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
+        # For simple cases, you can use the sync version
+        return await asyncio.to_thread(self.scrap, url, html, **kwargs)
+        
+
+    def _get_pdf_path(self, url: str) -> str:
+        if url.startswith(("http://", "https://")):
+            import tempfile
+            import requests
+            
+            # Create temp file with .pdf extension
+            temp_file = tempfile.NamedTemporaryFile(suffix='.pdf', delete=False)
+            
+            try:
+                # Download PDF with streaming
+                response = requests.get(url, stream=True)
+                response.raise_for_status()
+                
+                # Write to temp file
+                with open(temp_file.name, 'wb') as f:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        f.write(chunk)
+                        
+                return temp_file.name
+                
+            except Exception as e:
+                # Clean up temp file if download fails
+                Path(temp_file.name).unlink(missing_ok=True)
+                raise RuntimeError(f"Failed to download PDF from {url}: {str(e)}")
+                
+        elif url.startswith("file://"):
+            return url[7:]  # Strip file:// prefix
+            
+        return url  # Assume local path
+    
+
+__all__ = ["PDFCrawlerStrategy", "PDFContentScrapingStrategy"]

crawl4ai/processors/pdf/processor.py

@@ -0,0 +1,487 @@
+import logging
+import re
+from abc import ABC, abstractmethod
+from datetime import datetime
+from pathlib import Path
+from time import time
+from dataclasses import dataclass, asdict, field
+from typing import Dict, List, Optional, Any, Union
+import base64
+import tempfile
+from .utils import *
+from .utils import (
+    apply_png_predictor,
+    clean_pdf_text,
+    clean_pdf_text_to_html,
+)
+
+# Remove direct PyPDF2 imports from the top
+# import PyPDF2
+# from PyPDF2 import PdfReader
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class PDFMetadata:
+    title: Optional[str] = None
+    author: Optional[str] = None
+    producer: Optional[str] = None
+    created: Optional[datetime] = None
+    modified: Optional[datetime] = None
+    pages: int = 0
+    encrypted: bool = False
+    file_size: Optional[int] = None
+
+@dataclass
+class PDFPage:
+    page_number: int
+    raw_text: str = ""
+    markdown: str = ""
+    html: str = ""
+    images: List[Dict] = field(default_factory=list)
+    links: List[str] = field(default_factory=list)
+    layout: List[Dict] = field(default_factory=list)
+
+@dataclass
+class PDFProcessResult:
+    metadata: PDFMetadata
+    pages: List[PDFPage]
+    processing_time: float = 0.0
+    version: str = "1.0"
+
+class PDFProcessorStrategy(ABC):
+    @abstractmethod
+    def process(self, pdf_path: Path) -> PDFProcessResult:
+        pass
+
+class NaivePDFProcessorStrategy(PDFProcessorStrategy):
+    def __init__(self, image_dpi: int = 144, image_quality: int = 85, extract_images: bool = True, 
+                 save_images_locally: bool = False, image_save_dir: Optional[Path] = None, batch_size: int = 4):
+        # Import check at initialization time
+        try:
+            import PyPDF2
+        except ImportError:
+            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            
+        self.image_dpi = image_dpi
+        self.image_quality = image_quality
+        self.current_page_number = 0
+        self.extract_images = extract_images
+        self.save_images_locally = save_images_locally
+        self.image_save_dir = image_save_dir
+        self.batch_size = batch_size
+        self._temp_dir = None
+
+    def process(self, pdf_path: Path) -> PDFProcessResult:
+        # Import inside method to allow dependency to be optional
+        try:
+            from PyPDF2 import PdfReader
+        except ImportError:
+            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            
+        start_time = time()
+        result = PDFProcessResult(
+            metadata=PDFMetadata(),
+            pages=[],
+            version="1.1"
+        )
+
+        try:
+            with pdf_path.open('rb') as file:
+                reader = PdfReader(file)
+                result.metadata = self._extract_metadata(pdf_path, reader)
+                
+                # Handle image directory
+                image_dir = None
+                if self.extract_images and self.save_images_locally:
+                    if self.image_save_dir:
+                        image_dir = Path(self.image_save_dir)
+                        image_dir.mkdir(exist_ok=True, parents=True)
+                    else:
+                        self._temp_dir = tempfile.mkdtemp(prefix='pdf_images_')
+                        image_dir = Path(self._temp_dir)
+
+                for page_num, page in enumerate(reader.pages):
+                    self.current_page_number = page_num + 1
+                    pdf_page = self._process_page(page, image_dir)
+                    result.pages.append(pdf_page)
+
+        except Exception as e:
+            logger.error(f"Failed to process PDF: {str(e)}")
+            raise
+        finally:
+            # Cleanup temp directory if it was created
+            if self._temp_dir and not self.image_save_dir:
+                import shutil
+                try:
+                    shutil.rmtree(self._temp_dir)
+                except Exception as e:
+                    logger.error(f"Failed to cleanup temp directory: {str(e)}")
+
+        result.processing_time = time() - start_time
+        return result
+
+    def process_batch(self, pdf_path: Path) -> PDFProcessResult:
+        """Like process() but processes PDF pages in parallel batches"""
+        # Import inside method to allow dependency to be optional
+        try:
+            from PyPDF2 import PdfReader
+            import PyPDF2  # For type checking
+        except ImportError:
+            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            
+        import concurrent.futures
+        import threading
+        
+        # Initialize PyPDF2 thread support
+        if not hasattr(threading.current_thread(), "_children"): 
+            threading.current_thread()._children = set()
+        
+        start_time = time()
+        result = PDFProcessResult(
+            metadata=PDFMetadata(),
+            pages=[],
+            version="1.1" 
+        )
+
+        try:
+            # Get metadata and page count from main thread
+            with pdf_path.open('rb') as file:
+                reader = PdfReader(file)
+                result.metadata = self._extract_metadata(pdf_path, reader)
+                total_pages = len(reader.pages)
+
+            # Handle image directory setup
+            image_dir = None
+            if self.extract_images and self.save_images_locally:
+                if self.image_save_dir:
+                    image_dir = Path(self.image_save_dir)
+                    image_dir.mkdir(exist_ok=True, parents=True)
+                else:
+                    self._temp_dir = tempfile.mkdtemp(prefix='pdf_images_')
+                    image_dir = Path(self._temp_dir)
+
+            def process_page_safely(page_num: int):
+                # Each thread opens its own file handle
+                with pdf_path.open('rb') as file:
+                    thread_reader = PdfReader(file)
+                    page = thread_reader.pages[page_num]
+                    self.current_page_number = page_num + 1
+                    return self._process_page(page, image_dir)
+
+            # Process pages in parallel batches
+            with concurrent.futures.ThreadPoolExecutor(max_workers=self.batch_size) as executor:
+                futures = []
+                for page_num in range(total_pages):
+                    future = executor.submit(process_page_safely, page_num)
+                    futures.append((page_num + 1, future))
+
+                # Collect results in order
+                result.pages = [None] * total_pages
+                for page_num, future in futures:
+                    try:
+                        pdf_page = future.result()
+                        result.pages[page_num - 1] = pdf_page
+                    except Exception as e:
+                        logger.error(f"Failed to process page {page_num}: {str(e)}")
+                        raise
+
+        except Exception as e:
+            logger.error(f"Failed to process PDF: {str(e)}")
+            raise
+        finally:
+            # Cleanup temp directory if it was created
+            if self._temp_dir and not self.image_save_dir:
+                import shutil
+                try:
+                    shutil.rmtree(self._temp_dir)
+                except Exception as e:
+                    logger.error(f"Failed to cleanup temp directory: {str(e)}")
+
+        result.processing_time = time() - start_time
+        return result
+
+    def _process_page(self, page, image_dir: Optional[Path]) -> PDFPage:
+        pdf_page = PDFPage(
+            page_number=self.current_page_number,
+        )
+
+        # Text and font extraction
+        def visitor_text(text, cm, tm, font_dict, font_size):
+            pdf_page.raw_text += text
+            pdf_page.layout.append({
+                "type": "text",
+                "text": text,
+                "x": tm[4],
+                "y": tm[5],
+            })
+        
+        page.extract_text(visitor_text=visitor_text)
+
+        # Image extraction
+        if self.extract_images:
+            pdf_page.images = self._extract_images(page, image_dir)
+
+        # Link extraction
+        pdf_page.links = self._extract_links(page)
+        
+        # Add markdown content
+        pdf_page.markdown = clean_pdf_text(self.current_page_number, pdf_page.raw_text)
+        pdf_page.html = clean_pdf_text_to_html(self.current_page_number, pdf_page.raw_text)
+
+        return pdf_page
+
+    def _extract_images(self, page, image_dir: Optional[Path]) -> List[Dict]:
+        # Import PyPDF2 for type checking only when needed
+        try:
+            import PyPDF2
+        except ImportError:
+            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            
+        if not self.extract_images:
+            return []
+
+        images = []
+        try:
+            resources = page.get("/Resources")
+            if resources:  # Check if resources exist
+                resources = resources.get_object()  # Resolve IndirectObject
+                if '/XObject' in resources:
+                    xobjects = resources['/XObject'].get_object()
+                    img_count = 0
+                    for obj_name in xobjects:
+                        xobj = xobjects[obj_name]
+                        if hasattr(xobj, 'get_object') and callable(xobj.get_object):
+                            xobj = xobj.get_object()
+                            if xobj.get('/Subtype') == '/Image':
+                                try:
+                                    img_count += 1
+                                    img_filename = f"page_{self.current_page_number}_img_{img_count}"
+                                    data = xobj.get_data()
+                                    filters = xobj.get('/Filter', [])
+                                    if not isinstance(filters, list):
+                                        filters = [filters]
+
+                                    # Resolve IndirectObjects in properties
+                                    width = xobj.get('/Width', 0)
+                                    height = xobj.get('/Height', 0)
+                                    color_space = xobj.get('/ColorSpace', '/DeviceRGB')
+                                    if isinstance(color_space, PyPDF2.generic.IndirectObject):
+                                        color_space = color_space.get_object()
+
+                                    # Handle different image encodings
+                                    success = False
+                                    image_format = 'bin'
+                                    image_data = None
+                                    
+                                    if '/FlateDecode' in filters:
+                                        try:
+                                            decode_parms = xobj.get('/DecodeParms', {})
+                                            if isinstance(decode_parms, PyPDF2.generic.IndirectObject):
+                                                decode_parms = decode_parms.get_object()
+                                            
+                                            predictor = decode_parms.get('/Predictor', 1)
+                                            bits = xobj.get('/BitsPerComponent', 8)
+                                            colors = 3 if color_space == '/DeviceRGB' else 1
+
+                                            if predictor >= 10:
+                                                data = apply_png_predictor(data, width, bits, colors)
+
+                                            # Create PIL Image
+                                            from PIL import Image
+                                            mode = 'RGB' if color_space == '/DeviceRGB' else 'L'
+                                            img = Image.frombytes(mode, (width, height), data)
+                                            
+                                            if self.save_images_locally:
+                                                final_path = (image_dir / img_filename).with_suffix('.png')
+                                                img.save(final_path)
+                                                image_data = str(final_path)
+                                            else:
+                                                import io
+                                                img_byte_arr = io.BytesIO()
+                                                img.save(img_byte_arr, format='PNG')
+                                                image_data = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
+                                            
+                                            success = True
+                                            image_format = 'png'
+                                        except Exception as e:
+                                            logger.error(f"FlateDecode error: {str(e)}")
+
+                                    elif '/DCTDecode' in filters:
+                                        # JPEG image
+                                        try:
+                                            if self.save_images_locally:
+                                                final_path = (image_dir / img_filename).with_suffix('.jpg')
+                                                with open(final_path, 'wb') as f:
+                                                    f.write(data)
+                                                image_data = str(final_path)
+                                            else:
+                                                image_data = base64.b64encode(data).decode('utf-8')
+                                            success = True
+                                            image_format = 'jpeg'
+                                        except Exception as e:
+                                            logger.error(f"JPEG save error: {str(e)}")
+
+                                    elif '/CCITTFaxDecode' in filters:
+                                        try:
+                                            if data[:4] != b'II*\x00':
+                                                # Add TIFF header if missing
+                                                tiff_header = b'II*\x00\x08\x00\x00\x00\x0e\x00\x00\x01\x03\x00\x01\x00\x00\x00' + \
+                                                            width.to_bytes(4, 'little') + \
+                                                            b'\x01\x03\x00\x01\x00\x00\x00' + \
+                                                            height.to_bytes(4, 'little') + \
+                                                            b'\x01\x12\x00\x03\x00\x00\x00\x01\x00\x01\x00\x00\x01\x17\x00\x04\x00\x00\x00\x01\x00\x00\x00J\x01\x1B\x00\x05\x00\x00\x00\x01\x00\x00\x00R\x01\x28\x00\x03\x00\x00\x00\x01\x00\x02\x00\x00'
+                                                data = tiff_header + data
+                                            
+                                            if self.save_images_locally:
+                                                final_path = (image_dir / img_filename).with_suffix('.tiff')
+                                                with open(final_path, 'wb') as f:
+                                                    f.write(data)
+                                                image_data = str(final_path)
+                                            else:
+                                                image_data = base64.b64encode(data).decode('utf-8')
+                                            success = True
+                                            image_format = 'tiff'
+                                        except Exception as e:
+                                            logger.error(f"CCITT save error: {str(e)}")
+
+                                    elif '/JPXDecode' in filters:
+                                        # JPEG 2000
+                                        try:
+                                            if self.save_images_locally:
+                                                final_path = (image_dir / img_filename).with_suffix('.jp2')
+                                                with open(final_path, 'wb') as f:
+                                                    f.write(data)
+                                                image_data = str(final_path)
+                                            else:
+                                                image_data = base64.b64encode(data).decode('utf-8')
+                                            success = True
+                                            image_format = 'jpeg2000'
+                                        except Exception as e:
+                                            logger.error(f"JPEG2000 save error: {str(e)}")
+
+                                    if success and image_data:
+                                        image_info = {
+                                            "format": image_format,
+                                            "width": width,
+                                            "height": height,
+                                            "color_space": str(color_space),
+                                            "bits_per_component": xobj.get('/BitsPerComponent', 1)
+                                        }
+                                        
+                                        if self.save_images_locally:
+                                            image_info["path"] = image_data
+                                        else:
+                                            image_info["data"] = image_data
+                                            
+                                        images.append(image_info)
+                                    else:
+                                        # Fallback: Save raw data
+                                        if self.save_images_locally:
+                                            final_path = (image_dir / img_filename).with_suffix('.bin')
+                                            with open(final_path, 'wb') as f:
+                                                f.write(data)
+                                            logger.warning(f"Saved raw image data to {final_path}")
+                                        else:
+                                            image_data = base64.b64encode(data).decode('utf-8')
+                                            images.append({
+                                                "format": "bin",
+                                                "width": width,
+                                                "height": height,
+                                                "color_space": str(color_space),
+                                                "bits_per_component": xobj.get('/BitsPerComponent', 1),
+                                                "data": image_data
+                                            })
+
+                                except Exception as e:
+                                    logger.error(f"Error processing image: {str(e)}")
+        except Exception as e:
+            logger.error(f"Image extraction error: {str(e)}")
+        
+        return images
+
+    def _extract_links(self, page) -> List[str]:
+        links = []
+        if '/Annots' in page:
+            try:
+                for annot in page['/Annots']:
+                    a = annot.get_object()
+                    if '/A' in a and '/URI' in a['/A']:
+                        links.append(a['/A']['/URI'])
+            except Exception as e:
+                print(f"Link error: {str(e)}")
+        return links
+
+    def _extract_metadata(self, pdf_path: Path, reader = None) -> PDFMetadata:
+        # Import inside method to allow dependency to be optional 
+        if reader is None:
+            try:
+                from PyPDF2 import PdfReader
+                reader = PdfReader(pdf_path)
+            except ImportError:
+                raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+
+        meta = reader.metadata or {}
+        created = self._parse_pdf_date(meta.get('/CreationDate', ''))
+        modified = self._parse_pdf_date(meta.get('/ModDate', ''))
+        
+        return PDFMetadata(
+            title=meta.get('/Title'),
+            author=meta.get('/Author'),
+            producer=meta.get('/Producer'),
+            created=created,
+            modified=modified,
+            pages=len(reader.pages),
+            encrypted=reader.is_encrypted,
+            file_size=pdf_path.stat().st_size
+        )
+
+    def _parse_pdf_date(self, date_str: str) -> Optional[datetime]:
+        try:
+            match = re.match(r'D:(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})', date_str)
+            if not match:
+                return None
+                
+            return datetime(
+                year=int(match[1]),
+                month=int(match[2]),
+                day=int(match[3]),
+                hour=int(match[4]),
+                minute=int(match[5]),
+                second=int(match[6])
+            )
+        except:
+            return None
+
+# Usage example
+if __name__ == "__main__":
+    import json
+    from pathlib import Path
+    
+    try:
+        # Import PyPDF2 only when running the file directly
+        import PyPDF2
+        from PyPDF2 import PdfReader
+    except ImportError:
+        print("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+        exit(1)
+        
+    current_dir = Path(__file__).resolve().parent
+    pdf_path = f'{current_dir}/test.pdf'
+    
+    strategy = NaivePDFProcessorStrategy()
+    result = strategy.process(Path(pdf_path))
+    
+    # Convert to JSON
+    json_output = asdict(result)
+    print(json.dumps(json_output, indent=2, default=str))
+    
+    with open(f'{current_dir}/test.html', 'w') as f:
+        for page in result.pages:
+            f.write(f'<h1>Page {page["page_number"]}</h1>')
+            f.write(page['html'])
+    with open(f'{current_dir}/test.md', 'w') as f:
+        for page in result.pages:
+            f.write(f'# Page {page["page_number"]}\n\n')
+            f.write(clean_pdf_text(page["page_number"], page['raw_text']))
+            f.write('\n\n')

crawl4ai/processors/pdf/utils.py

@@ -0,0 +1,350 @@
+import re
+
+def apply_png_predictor(data, width, bits, color_channels):
+    """Decode PNG predictor (PDF 1.5+ filter)"""
+    bytes_per_pixel = (bits * color_channels) // 8
+    if (bits * color_channels) % 8 != 0:
+        bytes_per_pixel += 1
+        
+    stride = width * bytes_per_pixel
+    scanline_length = stride + 1  # +1 for filter byte
+    
+    if len(data) % scanline_length != 0:
+        raise ValueError("Invalid scanline structure")
+    
+    num_lines = len(data) // scanline_length
+    output = bytearray()
+    prev_line = b'\x00' * stride
+    
+    for i in range(num_lines):
+        line = data[i*scanline_length:(i+1)*scanline_length]
+        filter_type = line[0]
+        filtered = line[1:]
+        
+        if filter_type == 0:  # None
+            decoded = filtered
+        elif filter_type == 1:  # Sub
+            decoded = bytearray(filtered)
+            for j in range(bytes_per_pixel, len(decoded)):
+                decoded[j] = (decoded[j] + decoded[j - bytes_per_pixel]) % 256
+        elif filter_type == 2:  # Up
+            decoded = bytearray([(filtered[j] + prev_line[j]) % 256 
+                               for j in range(len(filtered))])
+        elif filter_type == 3:  # Average
+            decoded = bytearray(filtered)
+            for j in range(len(decoded)):
+                left = decoded[j - bytes_per_pixel] if j >= bytes_per_pixel else 0
+                up = prev_line[j]
+                avg = (left + up) // 2
+                decoded[j] = (decoded[j] + avg) % 256
+        elif filter_type == 4:  # Paeth
+            decoded = bytearray(filtered)
+            for j in range(len(decoded)):
+                left = decoded[j - bytes_per_pixel] if j >= bytes_per_pixel else 0
+                up = prev_line[j]
+                up_left = prev_line[j - bytes_per_pixel] if j >= bytes_per_pixel else 0
+                paeth = paeth_predictor(left, up, up_left)
+                decoded[j] = (decoded[j] + paeth) % 256
+        else:
+            raise ValueError(f"Unsupported filter type: {filter_type}")
+        
+        output.extend(decoded)
+        prev_line = decoded
+    
+    return bytes(output)
+
+def paeth_predictor(a, b, c):
+    p = a + b - c
+    pa = abs(p - a)
+    pb = abs(p - b)
+    pc = abs(p - c)
+    if pa <= pb and pa <= pc:
+        return a
+    elif pb <= pc:
+        return b
+    else:
+        return c
+
+import re
+import html
+
+def clean_pdf_text_to_html(page_number, text):
+    # Decode Unicode escapes and handle surrogate pairs
+    try:
+        decoded = text.encode('latin-1').decode('unicode-escape')
+        decoded = decoded.encode('utf-16', 'surrogatepass').decode('utf-16')
+    except Exception as e:
+        decoded = text  # Fallback if decoding fails
+    
+    article_title_detected = False
+    # decoded = re.sub(r'\.\n', '.\n\n', decoded)
+    # decoded = re.sub(r'\.\n', '<|break|>', decoded)
+    lines = decoded.split('\n')
+    output = []
+    current_paragraph = []
+    in_header = False
+    email_pattern = re.compile(r'\{.*?\}')
+    affiliation_pattern = re.compile(r'^†')
+    quote_pattern = re.compile(r'^["“]')
+    author_pattern = re.compile(
+        r'^\s*[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?'
+        r'(?:,\s*[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?)*'
+        r'(?:,\s*(?:and|&)\s+[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?)?\s*$'
+    )
+    
+    def flush_paragraph():
+        if current_paragraph:
+            para = ' '.join(current_paragraph)
+            para = re.sub(r'\s+', ' ', para).strip()
+            if para:
+                # escaped_para = html.escape(para)
+                escaped_para = para
+                # escaped_para = re.sub(r'\.\n', '.\n\n', escaped_para)
+                # Split escaped_para by <|break|> to avoid HTML escaping
+                escaped_para = escaped_para.split('.\n\n')
+                # Wrap each part in <p> tag
+                escaped_para = [f'<p>{part}</p>' for part in escaped_para]
+                output.append(f'<div class="paragraph">{"".join(escaped_para)}</div><hr/>')
+            current_paragraph.clear()
+    
+    for i, line in enumerate(lines):
+        line = line.strip()
+        
+        # Handle empty lines
+        if not line:
+            flush_paragraph()
+            continue
+            
+        # Detect article title (first line with reasonable length)
+        if not article_title_detected and i == 0 and 3 <= len(line.split()) <= 8 and len(lines) > 1:
+            flush_paragraph()
+            escaped_line = html.escape(line)
+            output.append(f'<h2>{escaped_line}</h2>')
+            article_title_detected = True
+            continue
+            
+        # Detect numbered headers like "2.1 Background"
+        numbered_header = re.match(r'^(\d+(?:\.\d+)*)\s+(.+)$', line)
+        if i > 0 and not lines[i-1].strip() and numbered_header:
+            flush_paragraph()
+            level = numbered_header.group(1).count('.') + 1
+            header_text = numbered_header.group(2)
+            md_level = min(level + 1, 6)
+            escaped_header = html.escape(header_text)
+            output.append(f'<h{md_level}>{escaped_header}</h{md_level}>')
+            in_header = True
+            continue
+            
+        # Detect authors
+        if page_number == 1 and author_pattern.match(line):
+            authors = re.sub(r'[†â€]', '', line)
+            authors = re.split(r', | and ', authors)
+            formatted_authors = []
+            for author in authors:
+                if author.strip():
+                    parts = [p for p in author.strip().split() if p]
+                    formatted = ' '.join(parts)
+                    escaped_author = html.escape(formatted)
+                    formatted_authors.append(f'<strong>{escaped_author}</strong>')
+            
+            if len(formatted_authors) > 1:
+                joined = ', '.join(formatted_authors[:-1]) + ' and ' + formatted_authors[-1]
+            else:
+                joined = formatted_authors[0]
+            
+            output.append(f'<p>{joined}</p>')
+            continue
+            
+        # Detect affiliation
+        if affiliation_pattern.match(line):
+            escaped_line = html.escape(line)
+            output.append(f'<p><em>{escaped_line}</em></p>')
+            continue
+            
+        # Detect emails
+        if email_pattern.match(line):
+            escaped_line = html.escape(line)
+            output.append(f'<p><code>{escaped_line}</code></p>')
+            continue
+            
+        # Detect section headers
+        if re.match(r'^(Abstract|\d+\s+[A-Z]|References|Appendix|Figure|Table)', line):
+            flush_paragraph()
+            escaped_line = html.escape(line)
+            output.append(f'<h2 class="section-header"><em>{escaped_line}</em></h2>')
+            in_header = True
+            continue
+            
+        # Handle quotes
+        if quote_pattern.match(line):
+            flush_paragraph()
+            escaped_line = html.escape(line)
+            output.append(f'<blockquote><p>{escaped_line}</p></blockquote>')
+            continue
+            
+        # Handle hyphenated words
+        if line.endswith('-'):
+            current_paragraph.append(line[:-1].strip())
+        else:
+            current_paragraph.append(line)
+            
+        # Handle paragraph breaks after headers
+        if in_header and not line.endswith(('.', '!', '?')):
+            flush_paragraph()
+            in_header = False
+    
+    flush_paragraph()
+    
+    # Post-process HTML
+    html_output = '\n'.join(output)
+    
+    # Fix common citation patterns
+    html_output = re.sub(r'\(([A-Z][a-z]+ et al\. \d{4})\)', r'<cite>\1</cite>', html_output)
+    
+    # Fix escaped characters
+    html_output = html_output.replace('\\ud835', '').replace('\\u2020', '†')
+    
+    # Remove leftover hyphens and fix spacing
+    html_output = re.sub(r'\s+-\s+', '', html_output)
+    html_output = re.sub(r'\s+([.,!?)])', r'\1', html_output)
+    
+    return html_output
+
+def clean_pdf_text(page_number, text):
+    # Decode Unicode escapes and handle surrogate pairs
+    try:
+        decoded = text.encode('latin-1').decode('unicode-escape')
+        decoded = decoded.encode('utf-16', 'surrogatepass').decode('utf-16')
+    except Exception as e:
+        decoded = text  # Fallback if decoding fails
+    
+    article_title_detected = False
+    decoded = re.sub(r'\.\n', '.\n\n', decoded)
+    lines = decoded.split('\n')
+    output = []
+    current_paragraph = []
+    in_header = False
+    email_pattern = re.compile(r'\{.*?\}')
+    affiliation_pattern = re.compile(r'^†')
+    quote_pattern = re.compile(r'^["“]')
+    author_pattern = re.compile(
+        r'^\s*[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?'
+        r'(?:,\s*[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?)*'
+        r'(?:,\s*(?:and|&)\s+[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?)?\s*$'
+    )
+    
+    def flush_paragraph():
+        if current_paragraph:
+            para = ' '.join(current_paragraph)
+            para = re.sub(r'\s+', ' ', para).strip()
+            if para:
+                output.append(para)
+            current_paragraph.clear()
+    
+    for i, line in enumerate(lines):
+        line = line.strip()
+        
+        # Handle special patterns
+        if not line:
+            flush_paragraph()
+            continue
+            
+        # Detect headline (first line, reasonable length, surrounded by empty lines)
+        if not article_title_detected and i == 0 and 3 <= len(line.split()) <= 8 and (len(lines) > 1):
+            flush_paragraph()
+            output.append(f'## {line}')
+            continue
+            
+        # Detect paragraph breaks for ALL paragraphs
+        if not line and current_paragraph:
+            flush_paragraph()
+            output.append('')  # Add empty line between paragraphs
+            continue
+                    
+        # Detect numbered headers like "2.1 Background"
+        numbered_header = re.match(r'^(\d+(?:\.\d+)*)\s+(.+)$', line)
+        if not lines[i-1].strip() and numbered_header:
+            flush_paragraph()
+            level = numbered_header.group(1).count('.') + 1  # Convert 2.1 → level 2
+            header_text = numbered_header.group(2)
+            # Never go beyond ### for subsections
+            md_level = min(level + 1, 6)   # 1 → ##, 2 → ###, 3 → #### etc
+            output.append(f'{"#" * md_level} {header_text}')
+            in_header = True
+            continue            
+            
+                    
+        # Detect authors
+        if page_number == 1 and author_pattern.match(line):
+            # Clean and format author names
+            authors = re.sub(r'[†â€]', '', line)  # Remove affiliation markers
+            authors = re.split(r', | and ', authors)
+            formatted_authors = []
+            for author in authors:
+                if author.strip():
+                    # Handle "First Last" formatting
+                    parts = [p for p in author.strip().split() if p]
+                    formatted = ' '.join(parts)
+                    formatted_authors.append(f'**{formatted}**')
+            
+            # Join with commas and "and"
+            if len(formatted_authors) > 1:
+                joined = ', '.join(formatted_authors[:-1]) + ' and ' + formatted_authors[-1]
+            else:
+                joined = formatted_authors[0]
+            
+            output.append(joined)
+            continue
+            
+        # Detect affiliation
+        if affiliation_pattern.match(line):
+            output.append(f'*{line}*')
+            continue
+            
+        # Detect emails
+        if email_pattern.match(line):
+            output.append(f'`{line}`')
+            continue
+            
+        # Detect section headers
+        if re.match(r'^(Abstract|\d+\s+[A-Z]|References|Appendix|Figure|Table)', line):
+            flush_paragraph()
+            output.append(f'_[{line}]_')
+            in_header = True
+            continue
+            
+           
+        # Handle quotes
+        if quote_pattern.match(line):
+            flush_paragraph()
+            output.append(f'> {line}')
+            continue
+            
+        # Handle hyphenated words
+        if line.endswith('-'):
+            current_paragraph.append(line[:-1].strip())
+        else:
+            current_paragraph.append(line)
+            
+        # Handle paragraph breaks after headers
+        if in_header and not line.endswith(('.', '!', '?')):
+            flush_paragraph()
+            in_header = False
+    
+    flush_paragraph()
+    
+    # Post-processing
+    markdown = '\n\n'.join(output)
+    
+    # Fix common citation patterns
+    markdown = re.sub(r'\(([A-Z][a-z]+ et al\. \d{4})\)', r'[\1]', markdown)
+    
+    # Fix escaped characters
+    markdown = markdown.replace('\\ud835', '').replace('\\u2020', '†')
+    
+    # Remove leftover hyphens and fix spacing
+    markdown = re.sub(r'\s+-\s+', '', markdown)  # Join hyphenated words
+    markdown = re.sub(r'\s+([.,!?)])', r'\1', markdown)  # Fix punctuation spacing
+    
+    
+    return markdown

crawl4ai/prompts.py

@@ -198,25 +198,70 @@ Avoid Common Mistakes:
 - Do NOT add any comments using "//" or "#" in the JSON output. It causes parsing errors.
 - Make sure the JSON is properly formatted with curly braces, square brackets, and commas in the right places.
 - Do not miss closing </blocks> tag at the end of the JSON output.
-- Do not generate the Python coee show me how to do the task, this is your task to extract the information and return it in JSON format.
+- Do not generate the Python code show me how to do the task, this is your task to extract the information and return it in JSON format.
 
 Result
 Output the final list of JSON objects, wrapped in <blocks>...</blocks> XML tags. Make sure to close the tag properly."""
 
+PROMPT_EXTRACT_INFERRED_SCHEMA = """Here is the content from the URL:
+<url>{URL}</url>
+
+<url_content>
+{HTML}
+</url_content>
+
+Please carefully read the URL content and the user's request. Analyze the page structure and infer the most appropriate JSON schema based on the content and request.
+
+Extraction Strategy:
+1. First, determine if the page contains repetitive items (like multiple products, articles, etc.) or a single content item (like a single article or page).
+2. For repetitive items: Identify the common pattern and extract each instance as a separate JSON object in an array.
+3. For single content: Extract the key information into a comprehensive JSON object that captures the essential details.
+
+Extraction instructions:
+Return the extracted information as a list of JSON objects. For repetitive content, each object in the list should correspond to a distinct item. For single content, you may return just one detailed JSON object. Wrap the entire JSON list in <blocks>...</blocks> XML tags.
+
+Schema Design Guidelines:
+- Create meaningful property names that clearly describe the data they contain
+- Use nested objects for hierarchical information
+- Use arrays for lists of related items
+- Include all information requested by the user
+- Maintain consistency in property names and data structures
+- Only include properties that are actually present in the content
+- For dates, prefer ISO format (YYYY-MM-DD)
+- For prices or numeric values, extract them without currency symbols when possible
+
+Quality Reflection:
+Before outputting your final answer, double check that:
+1. The inferred schema makes logical sense for the type of content
+2. All requested information is included
+3. The JSON is valid and could be parsed without errors
+4. Property names are consistent and descriptive
+5. The structure is optimal for the type of data being represented
+
+Avoid Common Mistakes:
+- Do NOT add any comments using "//" or "#" in the JSON output. It causes parsing errors.
+- Make sure the JSON is properly formatted with curly braces, square brackets, and commas in the right places.
+- Do not miss closing </blocks> tag at the end of the JSON output.
+- Do not generate Python code showing how to do the task; this is your task to extract the information and return it in JSON format.
+- Ensure consistency in property names across all objects
+- Don't include empty properties or null values unless they're meaningful
+- For repetitive content, ensure all objects follow the same schema
+
+Important: If user specific instruction is provided, then stress significantly on what user is requesting and describing about the schema of end result (if any). If user is requesting to extract specific information, then focus on that and ignore the rest of the content.
+<user_request>
+{REQUEST}
+</user_request>
+
+Result:
+Output the final list of JSON objects, wrapped in <blocks>...</blocks> XML tags. Make sure to close the tag properly.
+
+DO NOT ADD ANY PRE OR POST COMMENTS. JUST RETURN THE JSON OBJECTS INSIDE <blocks>...</blocks> TAGS.
+
+CRITICAL: The content inside the <blocks> tags MUST be a direct array of JSON objects (starting with '[' and ending with ']'), not a dictionary/object containing an array. For example, use <blocks>[{...}, {...}]</blocks> instead of <blocks>{"items": [{...}, {...}]}</blocks>. This is essential for proper parsing.
+"""
 
 PROMPT_FILTER_CONTENT = """Your task is to filter and convert HTML content into clean, focused markdown that's optimized for use with LLMs and information retrieval systems.
 
-INPUT HTML: 
-<|HTML_CONTENT_START|>
-{HTML}
-<|HTML_CONTENT_END|>
-
-
-SPECIFIC INSTRUCTION: 
-<|USER_INSTRUCTION_START|>
-{REQUEST}
-<|USER_INSTRUCTION_END|>
-
 TASK DETAILS:
 1. Content Selection
 - DO: Keep essential information, main content, key details
@@ -240,15 +285,7 @@ TASK DETAILS:
 - DON'T: Fragment related content
 - DON'T: Duplicate information
 
-Example Input:
-<div class="main-content"><h1>Setup Guide</h1><p>Follow these steps...</p></div>
-<div class="sidebar">Related articles...</div>
-
-Example Output:
-# Setup Guide
-Follow these steps...
-
-IMPORTANT: If specific instruction is provided above, prioritize those requirements over these general guidelines.
+IMPORTANT: If user specific instruction is provided, ignore above guideline and prioritize those requirements over these general guidelines.
 
 OUTPUT FORMAT: 
 Wrap your response in <content> tags. Use proper markdown throughout.
@@ -256,7 +293,18 @@ Wrap your response in <content> tags. Use proper markdown throughout.
 [Your markdown content here]
 </content>
 
-Begin filtering now."""
+Begin filtering now.
+
+--------------------------------------------
+
+<|HTML_CONTENT_START|>
+{HTML}
+<|HTML_CONTENT_END|>
+
+<|USER_INSTRUCTION_START|>
+{REQUEST}
+<|USER_INSTRUCTION_END|>
+"""
 
 JSON_SCHEMA_BUILDER= """
 # HTML Schema Generation Instructions

crawl4ai/proxy_strategy.py

@@ -0,0 +1,155 @@
+from typing import List, Dict, Optional
+from abc import ABC, abstractmethod
+from itertools import cycle
+import os
+
+
+class ProxyConfig:
+    def __init__(
+        self,
+        server: str,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        ip: Optional[str] = None,
+    ):
+        """Configuration class for a single proxy.
+        
+        Args:
+            server: Proxy server URL (e.g., "http://127.0.0.1:8080")
+            username: Optional username for proxy authentication
+            password: Optional password for proxy authentication
+            ip: Optional IP address for verification purposes
+        """
+        self.server = server
+        self.username = username
+        self.password = password
+        
+        # Extract IP from server if not explicitly provided
+        self.ip = ip or self._extract_ip_from_server()
+    
+    def _extract_ip_from_server(self) -> Optional[str]:
+        """Extract IP address from server URL."""
+        try:
+            # Simple extraction assuming http://ip:port format
+            if "://" in self.server:
+                parts = self.server.split("://")[1].split(":")
+                return parts[0]
+            else:
+                parts = self.server.split(":")
+                return parts[0]
+        except Exception:
+            return None
+    
+    @staticmethod
+    def from_string(proxy_str: str) -> "ProxyConfig":
+        """Create a ProxyConfig from a string in the format 'ip:port:username:password'."""
+        parts = proxy_str.split(":")
+        if len(parts) == 4:  # ip:port:username:password
+            ip, port, username, password = parts
+            return ProxyConfig(
+                server=f"http://{ip}:{port}",
+                username=username,
+                password=password,
+                ip=ip
+            )
+        elif len(parts) == 2:  # ip:port only
+            ip, port = parts
+            return ProxyConfig(
+                server=f"http://{ip}:{port}",
+                ip=ip
+            )
+        else:
+            raise ValueError(f"Invalid proxy string format: {proxy_str}")
+    
+    @staticmethod
+    def from_dict(proxy_dict: Dict) -> "ProxyConfig":
+        """Create a ProxyConfig from a dictionary."""
+        return ProxyConfig(
+            server=proxy_dict.get("server"),
+            username=proxy_dict.get("username"),
+            password=proxy_dict.get("password"),
+            ip=proxy_dict.get("ip")
+        )
+    
+    @staticmethod
+    def from_env(env_var: str = "PROXIES") -> List["ProxyConfig"]:
+        """Load proxies from environment variable.
+        
+        Args:
+            env_var: Name of environment variable containing comma-separated proxy strings
+            
+        Returns:
+            List of ProxyConfig objects
+        """
+        proxies = []
+        try:
+            proxy_list = os.getenv(env_var, "").split(",")
+            for proxy in proxy_list:
+                if not proxy:
+                    continue
+                proxies.append(ProxyConfig.from_string(proxy))
+        except Exception as e:
+            print(f"Error loading proxies from environment: {e}")
+        return proxies
+    
+    def to_dict(self) -> Dict:
+        """Convert to dictionary representation."""
+        return {
+            "server": self.server,
+            "username": self.username,
+            "password": self.password,
+            "ip": self.ip
+        }
+    
+    def clone(self, **kwargs) -> "ProxyConfig":
+        """Create a copy of this configuration with updated values.
+
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+
+        Returns:
+            ProxyConfig: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return ProxyConfig.from_dict(config_dict)
+
+
+class ProxyRotationStrategy(ABC):
+    """Base abstract class for proxy rotation strategies"""
+    
+    @abstractmethod
+    async def get_next_proxy(self) -> Optional[Dict]:
+        """Get next proxy configuration from the strategy"""
+        pass
+
+    @abstractmethod
+    def add_proxies(self, proxies: List[Dict]):
+        """Add proxy configurations to the strategy"""
+        pass
+
+class RoundRobinProxyStrategy:
+    """Simple round-robin proxy rotation strategy using ProxyConfig objects"""
+
+    def __init__(self, proxies: List[ProxyConfig] = None):
+        """
+        Initialize with optional list of proxy configurations
+        
+        Args:
+            proxies: List of ProxyConfig objects
+        """
+        self._proxies = []
+        self._proxy_cycle = None
+        if proxies:
+            self.add_proxies(proxies)
+
+    def add_proxies(self, proxies: List[ProxyConfig]):
+        """Add new proxies to the rotation pool"""
+        self._proxies.extend(proxies)
+        self._proxy_cycle = cycle(self._proxies)
+
+    async def get_next_proxy(self) -> Optional[ProxyConfig]:
+        """Get next proxy in round-robin fashion"""
+        if not self._proxy_cycle:
+            return None
+        return next(self._proxy_cycle)

crawl4ai/types.py

@@ -0,0 +1,187 @@
+from typing import TYPE_CHECKING, Union
+
+# Logger types
+AsyncLoggerBase = Union['AsyncLoggerBaseType']
+AsyncLogger = Union['AsyncLoggerType']
+
+# Crawler core types
+AsyncWebCrawler = Union['AsyncWebCrawlerType']
+CacheMode = Union['CacheModeType']
+CrawlResult = Union['CrawlResultType']
+CrawlerHub = Union['CrawlerHubType']
+BrowserProfiler = Union['BrowserProfilerType']
+
+# Configuration types
+BrowserConfig = Union['BrowserConfigType']
+CrawlerRunConfig = Union['CrawlerRunConfigType']
+HTTPCrawlerConfig = Union['HTTPCrawlerConfigType']
+LLMConfig = Union['LLMConfigType']
+
+# Content scraping types
+ContentScrapingStrategy = Union['ContentScrapingStrategyType']
+WebScrapingStrategy = Union['WebScrapingStrategyType']
+LXMLWebScrapingStrategy = Union['LXMLWebScrapingStrategyType']
+
+# Proxy types
+ProxyRotationStrategy = Union['ProxyRotationStrategyType']
+RoundRobinProxyStrategy = Union['RoundRobinProxyStrategyType']
+
+# Extraction types
+ExtractionStrategy = Union['ExtractionStrategyType']
+LLMExtractionStrategy = Union['LLMExtractionStrategyType']
+CosineStrategy = Union['CosineStrategyType']
+JsonCssExtractionStrategy = Union['JsonCssExtractionStrategyType']
+JsonXPathExtractionStrategy = Union['JsonXPathExtractionStrategyType']
+
+# Chunking types
+ChunkingStrategy = Union['ChunkingStrategyType']
+RegexChunking = Union['RegexChunkingType']
+
+# Markdown generation types
+DefaultMarkdownGenerator = Union['DefaultMarkdownGeneratorType']
+MarkdownGenerationResult = Union['MarkdownGenerationResultType']
+
+# Content filter types
+RelevantContentFilter = Union['RelevantContentFilterType']
+PruningContentFilter = Union['PruningContentFilterType']
+BM25ContentFilter = Union['BM25ContentFilterType']
+LLMContentFilter = Union['LLMContentFilterType']
+
+# Dispatcher types
+BaseDispatcher = Union['BaseDispatcherType']
+MemoryAdaptiveDispatcher = Union['MemoryAdaptiveDispatcherType']
+SemaphoreDispatcher = Union['SemaphoreDispatcherType']
+RateLimiter = Union['RateLimiterType']
+CrawlerMonitor = Union['CrawlerMonitorType']
+DisplayMode = Union['DisplayModeType']
+RunManyReturn = Union['RunManyReturnType']
+
+# Docker client
+Crawl4aiDockerClient = Union['Crawl4aiDockerClientType']
+
+# Deep crawling types
+DeepCrawlStrategy = Union['DeepCrawlStrategyType']
+BFSDeepCrawlStrategy = Union['BFSDeepCrawlStrategyType']
+FilterChain = Union['FilterChainType']
+ContentTypeFilter = Union['ContentTypeFilterType']
+DomainFilter = Union['DomainFilterType']
+URLFilter = Union['URLFilterType']
+FilterStats = Union['FilterStatsType']
+SEOFilter = Union['SEOFilterType']
+KeywordRelevanceScorer = Union['KeywordRelevanceScorerType']
+URLScorer = Union['URLScorerType']
+CompositeScorer = Union['CompositeScorerType']
+DomainAuthorityScorer = Union['DomainAuthorityScorerType']
+FreshnessScorer = Union['FreshnessScorerType']
+PathDepthScorer = Union['PathDepthScorerType']
+BestFirstCrawlingStrategy = Union['BestFirstCrawlingStrategyType']
+DFSDeepCrawlStrategy = Union['DFSDeepCrawlStrategyType']
+DeepCrawlDecorator = Union['DeepCrawlDecoratorType']
+
+# Only import types during type checking to avoid circular imports
+if TYPE_CHECKING:
+    # Logger imports
+    from .async_logger import (
+        AsyncLoggerBase as AsyncLoggerBaseType,
+        AsyncLogger as AsyncLoggerType,
+    )
+    
+    # Crawler core imports
+    from .async_webcrawler import (
+        AsyncWebCrawler as AsyncWebCrawlerType,
+        CacheMode as CacheModeType,
+    )
+    from .models import CrawlResult as CrawlResultType
+    from .hub import CrawlerHub as CrawlerHubType
+    from .browser_profiler import BrowserProfiler as BrowserProfilerType
+    
+    # Configuration imports
+    from .async_configs import (
+        BrowserConfig as BrowserConfigType,
+        CrawlerRunConfig as CrawlerRunConfigType,
+        HTTPCrawlerConfig as HTTPCrawlerConfigType,
+        LLMConfig as LLMConfigType,
+    )
+    
+    # Content scraping imports
+    from .content_scraping_strategy import (
+        ContentScrapingStrategy as ContentScrapingStrategyType,
+        WebScrapingStrategy as WebScrapingStrategyType,
+        LXMLWebScrapingStrategy as LXMLWebScrapingStrategyType,
+    )
+    
+    # Proxy imports
+    from .proxy_strategy import (
+        ProxyRotationStrategy as ProxyRotationStrategyType,
+        RoundRobinProxyStrategy as RoundRobinProxyStrategyType,
+    )
+    
+    # Extraction imports
+    from .extraction_strategy import (
+        ExtractionStrategy as ExtractionStrategyType,
+        LLMExtractionStrategy as LLMExtractionStrategyType,
+        CosineStrategy as CosineStrategyType,
+        JsonCssExtractionStrategy as JsonCssExtractionStrategyType,
+        JsonXPathExtractionStrategy as JsonXPathExtractionStrategyType,
+    )
+    
+    # Chunking imports
+    from .chunking_strategy import (
+        ChunkingStrategy as ChunkingStrategyType,
+        RegexChunking as RegexChunkingType,
+    )
+    
+    # Markdown generation imports
+    from .markdown_generation_strategy import (
+        DefaultMarkdownGenerator as DefaultMarkdownGeneratorType,
+    )
+    from .models import MarkdownGenerationResult as MarkdownGenerationResultType
+    
+    # Content filter imports
+    from .content_filter_strategy import (
+        RelevantContentFilter as RelevantContentFilterType,
+        PruningContentFilter as PruningContentFilterType,
+        BM25ContentFilter as BM25ContentFilterType,
+        LLMContentFilter as LLMContentFilterType,
+    )
+    
+    # Dispatcher imports
+    from .async_dispatcher import (
+        BaseDispatcher as BaseDispatcherType,
+        MemoryAdaptiveDispatcher as MemoryAdaptiveDispatcherType,
+        SemaphoreDispatcher as SemaphoreDispatcherType,
+        RateLimiter as RateLimiterType,
+        CrawlerMonitor as CrawlerMonitorType,
+        DisplayMode as DisplayModeType,
+        RunManyReturn as RunManyReturnType,
+    )
+    
+    # Docker client
+    from .docker_client import Crawl4aiDockerClient as Crawl4aiDockerClientType
+    
+    # Deep crawling imports
+    from .deep_crawling import (
+        DeepCrawlStrategy as DeepCrawlStrategyType,
+        BFSDeepCrawlStrategy as BFSDeepCrawlStrategyType,
+        FilterChain as FilterChainType,
+        ContentTypeFilter as ContentTypeFilterType,
+        DomainFilter as DomainFilterType,
+        URLFilter as URLFilterType,
+        FilterStats as FilterStatsType,
+        SEOFilter as SEOFilterType,
+        KeywordRelevanceScorer as KeywordRelevanceScorerType,
+        URLScorer as URLScorerType,
+        CompositeScorer as CompositeScorerType,
+        DomainAuthorityScorer as DomainAuthorityScorerType,
+        FreshnessScorer as FreshnessScorerType,
+        PathDepthScorer as PathDepthScorerType,
+        BestFirstCrawlingStrategy as BestFirstCrawlingStrategyType,
+        DFSDeepCrawlStrategy as DFSDeepCrawlStrategyType,
+        DeepCrawlDecorator as DeepCrawlDecoratorType,
+    )
+
+
+
+def create_llm_config(*args, **kwargs) -> 'LLMConfigType':
+    from .async_configs import LLMConfig
+    return LLMConfig(*args, **kwargs)

crawl4ai/user_agent_generator.py

@@ -3,12 +3,11 @@ from typing import Optional, Literal, List, Dict, Tuple
 import re
 
 from abc import ABC, abstractmethod
-import random
 from fake_useragent import UserAgent
 import requests
 from lxml import html
 import json
-from typing import Optional, List, Union, Dict
+from typing import Union
 
 class UAGen(ABC):
    @abstractmethod

crawl4ai/utils.py

@@ -1,20 +1,24 @@
 import time
-from urllib.parse import urlparse
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from bs4 import BeautifulSoup, Comment, element, Tag, NavigableString
 import json
 import html
+import lxml
 import re
 import os
 import platform
 from .prompts import PROMPT_EXTRACT_BLOCKS
-from .config import *
+from array import array
+from .html2text import html2text, CustomHTML2Text
+# from .config import *
+from .config import MIN_WORD_THRESHOLD, IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, IMAGE_SCORE_THRESHOLD, DEFAULT_PROVIDER, PROVIDER_MODELS
+import httpx
+from socket import gaierror
 from pathlib import Path
-from typing import Dict, Any
+from typing import Dict, Any, List, Optional, Callable
 from urllib.parse import urljoin
 import requests
 from requests.exceptions import InvalidSchema
-from typing import Dict, Any
 import xxhash
 from colorama import Fore, Style, init
 import textwrap
@@ -22,12 +26,191 @@ import cProfile
 import pstats
 from functools import wraps
 import asyncio
-
+from lxml import etree, html as lhtml
 import sqlite3
 import hashlib
-from urllib.parse import urljoin, urlparse
+
 from urllib.robotparser import RobotFileParser
 import aiohttp
+from urllib.parse import urlparse, urlunparse
+from functools import lru_cache
+
+from packaging import version
+from . import __version__
+from typing import Sequence
+
+from itertools import chain
+from collections import deque
+from typing import  Generator, Iterable
+
+def chunk_documents(
+    documents: Iterable[str],
+    chunk_token_threshold: int,
+    overlap: int,
+    word_token_rate: float = 0.75,
+    tokenizer: Optional[Callable[[str], List[str]]] = None,
+) -> Generator[str, None, None]:
+    """
+    Efficiently chunks documents into token-limited sections with overlap between chunks.
+
+    Args:
+        documents: Iterable of document strings
+        chunk_token_threshold: Maximum tokens per chunk
+        overlap: Number of tokens to overlap between chunks
+        word_token_rate: Token estimate per word when not using a tokenizer
+        tokenizer: Function that splits text into tokens (if available)
+
+    Yields:
+        Text chunks as strings
+    """
+    token_queue = deque()
+    contribution_queue = deque()
+    current_token_count = 0.0
+
+    for doc in documents:
+        # Tokenize document
+        if tokenizer:
+            tokens = tokenizer(doc)
+            contributions = [1.0] * len(tokens)
+        else:
+            tokens = doc.split()
+            contributions = [word_token_rate] * len(tokens)
+
+        # Add to processing queues
+        token_queue.extend(tokens)
+        contribution_queue.extend(contributions)
+        current_token_count += sum(contributions)
+
+        # Process full chunks
+        while current_token_count >= chunk_token_threshold:
+            # Find chunk split point
+            chunk_tokens = []
+            chunk_contrib = []
+            chunk_total = 0.0
+            
+            # Build chunk up to threshold
+            while contribution_queue:
+                next_contrib = contribution_queue[0]
+                if chunk_total + next_contrib > chunk_token_threshold:
+                    break
+                
+                chunk_total += next_contrib
+                chunk_contrib.append(contribution_queue.popleft())
+                chunk_tokens.append(token_queue.popleft())
+
+            # Handle edge case where first token exceeds threshold
+            if not chunk_contrib:  # Single token exceeds threshold
+                chunk_contrib.append(contribution_queue.popleft())
+                chunk_tokens.append(token_queue.popleft())
+
+            # Calculate overlap
+            overlap_total = 0.0
+            overlap_idx = 0
+            for contrib in reversed(chunk_contrib):
+                if overlap_total + contrib > overlap:
+                    break
+                overlap_total += contrib
+                overlap_idx += 1
+
+            # Prepend overlap to queues
+            if overlap_idx > 0:
+                overlap_tokens = chunk_tokens[-overlap_idx:]
+                overlap_contrib = chunk_contrib[-overlap_idx:]
+                
+                token_queue.extendleft(reversed(overlap_tokens))
+                contribution_queue.extendleft(reversed(overlap_contrib))
+                current_token_count += overlap_total
+
+            # Update current token count and yield chunk
+            current_token_count -= sum(chunk_contrib)
+            yield " ".join(chunk_tokens[:len(chunk_tokens)-overlap_idx] if overlap_idx else chunk_tokens)
+
+    # Yield remaining tokens
+    if token_queue:
+        yield " ".join(token_queue)
+
+def merge_chunks(
+    docs: Sequence[str], 
+    target_size: int,
+    overlap: int = 0,
+    word_token_ratio: float = 1.0,
+    splitter: Callable = None
+) -> List[str]:
+    """Merges documents into chunks of specified token size.
+    
+    Args:
+        docs: Input documents
+        target_size: Desired token count per chunk
+        overlap: Number of tokens to overlap between chunks
+        word_token_ratio: Multiplier for word->token conversion
+    """
+    # Pre-tokenize all docs and store token counts
+    splitter = splitter or str.split
+    token_counts = array('I')
+    all_tokens: List[List[str]] = []
+    total_tokens = 0
+    
+    for doc in docs:
+        tokens = doc.split()
+        count = int(len(tokens) * word_token_ratio)
+        if count:  # Skip empty docs
+            token_counts.append(count)
+            all_tokens.append(tokens)
+            total_tokens += count
+    
+    if not total_tokens:
+        return []
+
+    # Pre-allocate chunks
+    num_chunks = max(1, (total_tokens + target_size - 1) // target_size)
+    chunks: List[List[str]] = [[] for _ in range(num_chunks)]
+    
+    curr_chunk = 0
+    curr_size = 0
+    
+    # Distribute tokens
+    for tokens in chain.from_iterable(all_tokens):
+        if curr_size >= target_size and curr_chunk < num_chunks - 1:
+            if overlap > 0:
+                overlap_tokens = chunks[curr_chunk][-overlap:]
+                curr_chunk += 1
+                chunks[curr_chunk].extend(overlap_tokens)
+                curr_size = len(overlap_tokens)
+            else:
+                curr_chunk += 1
+                curr_size = 0
+                
+        chunks[curr_chunk].append(tokens)
+        curr_size += 1
+
+    # Return only non-empty chunks
+    return [' '.join(chunk) for chunk in chunks if chunk]
+
+
+class VersionManager:
+    def __init__(self):
+        self.home_dir = Path.home() / ".crawl4ai"
+        self.version_file = self.home_dir / "version.txt"
+
+    def get_installed_version(self):
+        """Get the version recorded in home directory"""
+        if not self.version_file.exists():
+            return None
+        try:
+            return version.parse(self.version_file.read_text().strip())
+        except Exception as _ex:
+            return None
+
+    def update_version(self):
+        """Update the version file to current library version"""
+        self.version_file.write_text(__version__.__version__)
+
+    def needs_update(self):
+        """Check if database needs update based on version"""
+        installed = self.get_installed_version()
+        current = version.parse(__version__.__version__)
+        return installed is None or installed < current
+
 
 class RobotsParser:
     # Default 7 days cache TTL
@@ -107,7 +290,7 @@ class RobotsParser:
             domain = parsed.netloc
             if not domain:
                 return True
-        except:
+        except Exception as _ex:
             return True
 
         # Fast path - check cache first
@@ -127,7 +310,7 @@ class RobotsParser:
                             self._cache_rules(domain, rules)
                         else:
                             return True
-            except:
+            except Exception as _ex:
                 # On any error (timeout, connection failed, etc), allow access
                 return True
 
@@ -160,6 +343,77 @@ class InvalidCSSSelectorError(Exception):
     pass
 
 
+SPLITS = bytearray([
+    # Control chars (0-31) + space (32)
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    # Special chars (33-47): ! " # $ % & ' ( ) * + , - . /
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+    # Numbers (48-57): Treat as non-splits
+    0,0,0,0,0,0,0,0,0,0,
+    # More special chars (58-64): : ; < = > ? @
+    1,1,1,1,1,1,1,
+    # Uppercase (65-90): Keep
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    # More special chars (91-96): [ \ ] ^ _ `
+    1,1,1,1,1,1,
+    # Lowercase (97-122): Keep
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    # Special chars (123-126): { | } ~
+    1,1,1,1,
+    # Extended ASCII
+    *([1] * 128)
+])
+
+# Additional split chars for HTML/code
+HTML_CODE_CHARS = {
+    # HTML specific
+    '•', '►', '▼', '©', '®', '™', '→', '⇒', '≈', '≤', '≥',
+    # Programming symbols  
+    '+=', '-=', '*=', '/=', '=>', '<=>', '!=', '==', '===',
+    '++', '--', '<<', '>>', '&&', '||', '??', '?:', '?.', 
+    # Common Unicode
+    '…', '"', '"', ''', ''', '«', '»', '—', '–',
+    # Additional splits
+    '+', '=', '~', '@', '#', '$', '%', '^', '&', '*',
+    '(', ')', '{', '}', '[', ']', '|', '\\', '/', '`',
+    '<', '>', ',', '.', '?', '!', ':', ';', '-', '_'
+}
+
+def advanced_split(text: str) -> list[str]:
+    result = []
+    word = array('u')
+    
+    i = 0
+    text_len = len(text)
+    
+    while i < text_len:
+        char = text[i]
+        o = ord(char)
+        
+        # Fast path for ASCII
+        if o < 256 and SPLITS[o]:
+            if word:
+                result.append(word.tounicode())
+                word = array('u')
+        # Check for multi-char symbols
+        elif i < text_len - 1:
+            two_chars = char + text[i + 1]
+            if two_chars in HTML_CODE_CHARS:
+                if word:
+                    result.append(word.tounicode())
+                    word = array('u')
+                i += 1  # Skip next char since we used it
+            else:
+                word.append(char)
+        else:
+            word.append(char)
+        i += 1
+            
+    if word:
+        result.append(word.tounicode())
+        
+    return result
+
 def create_box_message(
     message: str,
     type: str = "info",
@@ -1124,7 +1378,7 @@ def get_content_of_website_optimized(
             src = img.get("src", "")
             if base64_pattern.match(src):
                 img["src"] = base64_pattern.sub("", src)
-        except:
+        except Exception as _ex:
             pass
 
     cleaned_html = str(body).replace("\n\n", "\n").replace("  ", " ")
@@ -1162,7 +1416,7 @@ def extract_metadata_using_lxml(html, doc=None):
 
     if doc is None:
         try:
-            doc = lhtml.document_fromstring(html)
+            doc = lxml.html.document_fromstring(html)
         except Exception:
             return {}
 
@@ -1297,7 +1551,7 @@ def extract_xml_tags(string):
     return list(set(tags))
 
 
-def extract_xml_data(tags, string):
+def extract_xml_data_legacy(tags, string):
     """
     Extract data for specified XML tags from a string.
 
@@ -1326,6 +1580,38 @@ def extract_xml_data(tags, string):
 
     return data
 
+def extract_xml_data(tags, string):
+    """
+    Extract data for specified XML tags from a string, returning the longest content for each tag.
+
+    How it works:
+    1. Finds all occurrences of each tag in the string using regex.
+    2. For each tag, selects the occurrence with the longest content.
+    3. Returns a dictionary of tag-content pairs.
+
+    Args:
+        tags (List[str]): The list of XML tags to extract.
+        string (str): The input string containing XML data.
+
+    Returns:
+        Dict[str, str]: A dictionary with tag names as keys and longest extracted content as values.
+    """
+
+    data = {}
+
+    for tag in tags:
+        pattern = f"<{tag}>(.*?)</{tag}>"
+        matches = re.findall(pattern, string, re.DOTALL)
+        
+        if matches:
+            # Find the longest content for this tag
+            longest_content = max(matches, key=len).strip()
+            data[tag] = longest_content
+        else:
+            data[tag] = ""
+
+    return data
+
 
 def perform_completion_with_backoff(
     provider,
@@ -1394,6 +1680,19 @@ def perform_completion_with_backoff(
                         "content": ["Rate limit error. Please try again later."],
                     }
                 ]
+        except Exception as e:
+            raise e  # Raise any other exceptions immediately
+            # print("Error during completion request:", str(e))
+            # error_message = e.message
+            # return [
+            #     {
+            #         "index": 0,
+            #         "tags": ["error"],
+            #         "content": [
+            #             f"Error during LLM completion request. {error_message}"
+            #         ],
+            #     }
+            # ]
 
 
 def extract_blocks(url, html, provider=DEFAULT_PROVIDER, api_token=None, base_url=None):
@@ -1478,10 +1777,10 @@ def extract_blocks_batch(batch_data, provider="groq/llama3-70b-8192", api_token=
 
     messages = []
 
-    for url, html in batch_data:
+    for url, _html in batch_data:
         variable_values = {
             "URL": url,
-            "HTML": html,
+            "HTML": _html,
         }
 
         prompt_with_variables = PROMPT_EXTRACT_BLOCKS
@@ -1663,7 +1962,7 @@ def fast_format_html(html_string):
     indent = 0
     indent_str = "  "  # Two spaces for indentation
     formatted = []
-    in_content = False
+    # in_content = False
 
     # Split by < and > to separate tags and content
     parts = html_string.replace(">", ">\n").replace("<", "\n<").split("\n")
@@ -1709,6 +2008,82 @@ def normalize_url(href, base_url):
     return normalized
 
 
+def normalize_url_for_deep_crawl(href, base_url):
+    """Normalize URLs to ensure consistent format"""
+    from urllib.parse import urljoin, urlparse, urlunparse, parse_qs, urlencode
+
+    # Handle None or empty values
+    if not href:
+        return None
+
+    # Use urljoin to handle relative URLs
+    full_url = urljoin(base_url, href.strip())
+    
+    # Parse the URL for normalization
+    parsed = urlparse(full_url)
+    
+    # Convert hostname to lowercase
+    netloc = parsed.netloc.lower()
+    
+    # Remove fragment entirely
+    fragment = ''
+    
+    # Normalize query parameters if needed
+    query = parsed.query
+    if query:
+        # Parse query parameters
+        params = parse_qs(query)
+        
+        # Remove tracking parameters (example - customize as needed)
+        tracking_params = ['utm_source', 'utm_medium', 'utm_campaign', 'ref', 'fbclid']
+        for param in tracking_params:
+            if param in params:
+                del params[param]
+                
+        # Rebuild query string, sorted for consistency
+        query = urlencode(params, doseq=True) if params else ''
+    
+    # Build normalized URL
+    normalized = urlunparse((
+        parsed.scheme,
+        netloc,
+        parsed.path.rstrip('/') or '/',  # Normalize trailing slash
+        parsed.params,
+        query,
+        fragment
+    ))
+    
+    return normalized
+
+@lru_cache(maxsize=10000)
+def efficient_normalize_url_for_deep_crawl(href, base_url):
+    """Efficient URL normalization with proper parsing"""
+    from urllib.parse import urljoin
+    
+    if not href:
+        return None
+    
+    # Resolve relative URLs
+    full_url = urljoin(base_url, href.strip())
+    
+    # Use proper URL parsing
+    parsed = urlparse(full_url)
+    
+    # Only perform the most critical normalizations
+    # 1. Lowercase hostname
+    # 2. Remove fragment
+    normalized = urlunparse((
+        parsed.scheme,
+        parsed.netloc.lower(),
+        parsed.path,
+        parsed.params,
+        parsed.query,
+        ''  # Remove fragment
+    ))
+    
+    return normalized
+
+
 def normalize_url_tmp(href, base_url):
     """Normalize URLs to ensure consistent format"""
     # Extract protocol and domain from base URL
@@ -2207,3 +2582,196 @@ def get_error_context(exc_info, context_lines: int = 5):
         "function": func_name,
         "code_context": code_context,
     }
+
+def truncate(value, threshold):
+    if len(value) > threshold:
+        return value[:threshold] + '...'  # Add ellipsis to indicate truncation
+    return value
+
+def optimize_html(html_str, threshold=200):
+    root = lxml.html.fromstring(html_str)
+    
+    for _element in root.iter():
+        # Process attributes
+        for attr in list(_element.attrib):
+            _element.attrib[attr] = truncate(_element.attrib[attr], threshold)
+        
+        # Process text content
+        if _element.text and len(_element.text) > threshold:
+            _element.text = truncate(_element.text, threshold)
+            
+        # Process tail text
+        if _element.tail and len(_element.tail) > threshold:
+            _element.tail = truncate(_element.tail, threshold)
+    
+    return lxml.html.tostring(root, encoding='unicode', pretty_print=False)
+
+class HeadPeekr:
+    @staticmethod
+    async def fetch_head_section(url, timeout=0.3):
+        headers = {
+            "User-Agent": "Mozilla/5.0 (compatible; CrawlBot/1.0)",
+            "Accept": "text/html",
+            "Connection": "close"  # Force close after response
+        }
+        try:
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                response = await client.get(url, headers=headers, follow_redirects=True)
+                
+                # Handle redirects explicitly by using the final URL
+                if response.url != url:
+                    url = str(response.url)
+                    response = await client.get(url, headers=headers)
+                
+                content = b""
+                async for chunk in response.aiter_bytes():
+                    content += chunk
+                    if b"</head>" in content:
+                        break  # Stop after detecting </head>
+                return content.split(b"</head>")[0] + b"</head>"
+        except (httpx.HTTPError, gaierror) :
+            return None
+
+    @staticmethod
+    async def peek_html(url, timeout=0.3):
+        head_section = await HeadPeekr.fetch_head_section(url, timeout=timeout)
+        if head_section:
+            return head_section.decode("utf-8", errors="ignore")
+        return None
+
+    @staticmethod
+    def extract_meta_tags(head_content: str):
+        meta_tags = {}
+        
+        # Find all meta tags
+        meta_pattern = r'<meta[^>]+>'
+        for meta_tag in re.finditer(meta_pattern, head_content):
+            tag = meta_tag.group(0)
+            
+            # Extract name/property and content
+            name_match = re.search(r'name=["\'](.*?)["\']', tag)
+            property_match = re.search(r'property=["\'](.*?)["\']', tag)
+            content_match = re.search(r'content=["\'](.*?)["\']', tag)
+            
+            if content_match and (name_match or property_match):
+                key = name_match.group(1) if name_match else property_match.group(1)
+                meta_tags[key] = content_match.group(1)
+                
+        return meta_tags
+
+    def get_title(head_content: str):
+        title_match = re.search(r'<title>(.*?)</title>', head_content, re.IGNORECASE | re.DOTALL)
+        return title_match.group(1) if title_match else None
+
+def preprocess_html_for_schema(html_content, text_threshold=100, attr_value_threshold=200, max_size=100000):
+    """
+    Preprocess HTML to reduce size while preserving structure for schema generation.
+    
+    Args:
+        html_content (str): Raw HTML content
+        text_threshold (int): Maximum length for text nodes before truncation
+        attr_value_threshold (int): Maximum length for attribute values before truncation
+        max_size (int): Target maximum size for output HTML
+        
+    Returns:
+        str: Preprocessed HTML content
+    """
+    try:
+        # Parse HTML with error recovery
+        parser = etree.HTMLParser(remove_comments=True, remove_blank_text=True)
+        tree = lhtml.fromstring(html_content, parser=parser)
+        
+        # 1. Remove HEAD section (keep only BODY)
+        head_elements = tree.xpath('//head')
+        for head in head_elements:
+            if head.getparent() is not None:
+                head.getparent().remove(head)
+        
+        # 2. Define tags to remove completely
+        tags_to_remove = [
+            'script', 'style', 'noscript', 'iframe', 'canvas', 'svg',
+            'video', 'audio', 'source', 'track', 'map', 'area'
+        ]
+        
+        # Remove unwanted elements
+        for tag in tags_to_remove:
+            elements = tree.xpath(f'//{tag}')
+            for element in elements:
+                if element.getparent() is not None:
+                    element.getparent().remove(element)
+        
+        # 3. Process remaining elements to clean attributes and truncate text
+        for element in tree.iter():
+            # Skip if we're at the root level
+            if element.getparent() is None:
+                continue
+                
+            # Clean non-essential attributes but preserve structural ones
+            # attribs_to_keep = {'id', 'class', 'name', 'href', 'src', 'type', 'value', 'data-'}
+
+            # This is more aggressive than the previous version
+            attribs_to_keep = {'id', 'class', 'name', 'type', 'value'}
+
+            # attributes_hates_truncate = ['id', 'class', "data-"]
+
+            # This means, I don't care, if an attribute is too long, truncate it, go and find a better css selector to build a schema
+            attributes_hates_truncate = []
+            
+            # Process each attribute
+            for attrib in list(element.attrib.keys()):
+                # Keep if it's essential or starts with data-
+                if not (attrib in attribs_to_keep or attrib.startswith('data-')):
+                    element.attrib.pop(attrib)
+                # Truncate long attribute values except for selectors
+                elif attrib not in attributes_hates_truncate and len(element.attrib[attrib]) > attr_value_threshold:
+                    element.attrib[attrib] = element.attrib[attrib][:attr_value_threshold] + '...'
+            
+            # Truncate text content if it's too long
+            if element.text and len(element.text.strip()) > text_threshold:
+                element.text = element.text.strip()[:text_threshold] + '...'
+                
+            # Also truncate tail text if present
+            if element.tail and len(element.tail.strip()) > text_threshold:
+                element.tail = element.tail.strip()[:text_threshold] + '...'
+        
+        # 4. Find repeated patterns and keep only a few examples
+        # This is a simplistic approach - more sophisticated pattern detection could be implemented
+        pattern_elements = {}
+        for element in tree.xpath('//*[contains(@class, "")]'):
+            parent = element.getparent()
+            if parent is None:
+                continue
+                
+            # Create a signature based on tag and classes
+            classes = element.get('class', '')
+            if not classes:
+                continue
+            signature = f"{element.tag}.{classes}"
+            
+            if signature in pattern_elements:
+                pattern_elements[signature].append(element)
+            else:
+                pattern_elements[signature] = [element]
+        
+        # Keep only 3 examples of each repeating pattern
+        for signature, elements in pattern_elements.items():
+            if len(elements) > 3:
+                # Keep the first 2 and last elements
+                for element in elements[2:-1]:
+                    if element.getparent() is not None:
+                        element.getparent().remove(element)
+        
+        # 5. Convert back to string
+        result = etree.tostring(tree, encoding='unicode', method='html')
+        
+        # If still over the size limit, apply more aggressive truncation
+        if len(result) > max_size:
+            return result[:max_size] + "..."
+            
+        return result
+    
+    except Exception as e:
+        # Fallback for parsing errors
+        return html_content[:max_size] if len(html_content) > max_size else html_content
+    
+

deploy/docker/.dockerignore

@@ -0,0 +1,31 @@
+# .dockerignore
+*
+
+# Allow specific files and directories when using local installation
+!crawl4ai/
+!docs/
+!deploy/docker/
+!setup.py
+!pyproject.toml
+!README.md
+!LICENSE
+!MANIFEST.in
+!setup.cfg
+!mkdocs.yml
+
+.git/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.DS_Store
+.env
+.venv
+venv/
+tests/
+coverage.xml
+*.log
+*.swp
+*.egg-info/
+dist/
+build/

deploy/docker/.llm.env.example

@@ -0,0 +1,8 @@
+# LLM Provider Keys
+OPENAI_API_KEY=your_openai_key_here
+DEEPSEEK_API_KEY=your_deepseek_key_here
+ANTHROPIC_API_KEY=your_anthropic_key_here
+GROQ_API_KEY=your_groq_key_here
+TOGETHER_API_KEY=your_together_key_here
+MISTRAL_API_KEY=your_mistral_key_here
+GEMINI_API_TOKEN=your_gemini_key_here

deploy/docker/README.md

@@ -0,0 +1,833 @@
+# Crawl4AI Docker Guide 🐳
+
+## Table of Contents
+- [Prerequisites](#prerequisites)
+- [Installation](#installation)
+  - [Local Build](#local-build)
+  - [Docker Hub](#docker-hub)
+- [Dockerfile Parameters](#dockerfile-parameters)
+- [Using the API](#using-the-api)
+  - [Understanding Request Schema](#understanding-request-schema)
+  - [REST API Examples](#rest-api-examples)
+  - [Python SDK](#python-sdk)
+- [Metrics & Monitoring](#metrics--monitoring)
+- [Deployment Scenarios](#deployment-scenarios)
+- [Complete Examples](#complete-examples)
+- [Getting Help](#getting-help)
+
+## Prerequisites
+
+Before we dive in, make sure you have:
+- Docker installed and running (version 20.10.0 or higher)
+- At least 4GB of RAM available for the container
+- Python 3.10+ (if using the Python SDK)
+- Node.js 16+ (if using the Node.js examples)
+
+> 💡 **Pro tip**: Run `docker info` to check your Docker installation and available resources.
+
+## Installation
+
+### Local Build
+
+Let's get your local environment set up step by step!
+
+#### 1. Building the Image
+
+First, clone the repository and build the Docker image:
+
+```bash
+# Clone the repository
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai/deploy
+
+# Build the Docker image
+docker build --platform=linux/amd64 --no-cache -t crawl4ai .
+
+# Or build for arm64
+docker build --platform=linux/arm64 --no-cache -t crawl4ai .
+```
+
+#### 2. Environment Setup
+
+If you plan to use LLMs (Language Models), you'll need to set up your API keys. Create a `.llm.env` file:
+
+```env
+# OpenAI
+OPENAI_API_KEY=sk-your-key
+
+# Anthropic
+ANTHROPIC_API_KEY=your-anthropic-key
+
+# DeepSeek
+DEEPSEEK_API_KEY=your-deepseek-key
+
+# Check out https://docs.litellm.ai/docs/providers for more providers!
+```
+
+> 🔑 **Note**: Keep your API keys secure! Never commit them to version control.
+
+#### 3. Running the Container
+
+You have several options for running the container:
+
+Basic run (no LLM support):
+```bash
+docker run -d -p 8000:8000 --name crawl4ai crawl4ai
+```
+
+With LLM support:
+```bash
+docker run -d -p 8000:8000 \
+  --env-file .llm.env \
+  --name crawl4ai \
+  crawl4ai
+```
+
+Using host environment variables (Not a good practice, but works for local testing):
+```bash
+docker run -d -p 8000:8000 \
+  --env-file .llm.env \
+  --env "$(env)" \
+  --name crawl4ai \
+  crawl4ai
+```
+
+#### Multi-Platform Build
+For distributing your image across different architectures, use `buildx`:
+
+```bash
+# Set up buildx builder
+docker buildx create --use
+
+# Build for multiple platforms
+docker buildx build \
+  --platform linux/amd64,linux/arm64 \
+  -t crawl4ai \
+  --push \
+  .
+```
+
+> 💡 **Note**: Multi-platform builds require Docker Buildx and need to be pushed to a registry.
+
+#### Development Build
+For development, you might want to enable all features:
+
+```bash
+docker build -t crawl4ai
+  --build-arg INSTALL_TYPE=all \
+  --build-arg PYTHON_VERSION=3.10 \
+  --build-arg ENABLE_GPU=true \
+  .
+```
+
+#### GPU-Enabled Build
+If you plan to use GPU acceleration:
+
+```bash
+docker build -t crawl4ai
+  --build-arg ENABLE_GPU=true \
+  deploy/docker/
+```
+
+### Build Arguments Explained
+
+| Argument | Description | Default | Options |
+|----------|-------------|---------|----------|
+| PYTHON_VERSION | Python version | 3.10 | 3.8, 3.9, 3.10 |
+| INSTALL_TYPE | Feature set | default | default, all, torch, transformer |
+| ENABLE_GPU | GPU support | false | true, false |
+| APP_HOME | Install path | /app | any valid path |
+
+### Build Best Practices
+
+1. **Choose the Right Install Type**
+   - `default`: Basic installation, smallest image, to be honest, I use this most of the time.
+   - `all`: Full features, larger image (include transformer, and nltk, make sure you really need them)
+
+2. **Platform Considerations**
+   - Let Docker auto-detect platform unless you need cross-compilation
+   - Use --platform for specific architecture requirements
+   - Consider buildx for multi-architecture distribution
+
+3. **Performance Optimization**
+   - The image automatically includes platform-specific optimizations
+   - AMD64 gets OpenMP optimizations
+   - ARM64 gets OpenBLAS optimizations
+
+### Docker Hub
+
+> 🚧 Coming soon! The image will be available at `crawl4ai`. Stay tuned!
+
+## Using the API
+
+In the following sections, we discuss two ways to communicate with the Docker server. One option is to use the client SDK that I developed for Python, and I will soon develop one for Node.js. I highly recommend this approach to avoid mistakes. Alternatively, you can take a more technical route by using the JSON structure and passing it to all the URLs, which I will explain in detail.
+
+### Python SDK
+
+The SDK makes things easier! Here's how to use it:
+
+```python
+from crawl4ai.docker_client import Crawl4aiDockerClient
+from crawl4ai import BrowserConfig, CrawlerRunConfig
+
+async def main():
+    async with Crawl4aiDockerClient(base_url="http://localhost:8000", verbose=True) as client:
+      # If JWT is enabled, you can authenticate like this: (more on this later)
+        # await client.authenticate("test@example.com")
+        
+        # Non-streaming crawl
+        results = await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig()
+        )
+        print(f"Non-streaming results: {results}")
+        
+        # Streaming crawl
+        crawler_config = CrawlerRunConfig(stream=True)
+        async for result in await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=crawler_config
+        ):
+            print(f"Streamed result: {result}")
+        
+        # Get schema
+        schema = await client.get_schema()
+        print(f"Schema: {schema}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+`Crawl4aiDockerClient` is an async context manager that handles the connection for you. You can pass in optional parameters for more control:
+
+- `base_url` (str): Base URL of the Crawl4AI Docker server
+- `timeout` (float): Default timeout for requests in seconds
+- `verify_ssl` (bool): Whether to verify SSL certificates
+- `verbose` (bool): Whether to show logging output
+- `log_file` (str, optional): Path to log file if file logging is desired
+
+This client SDK generates a properly structured JSON request for the server's HTTP API.
+
+## Second Approach: Direct API Calls
+
+This is super important! The API expects a specific structure that matches our Python classes. Let me show you how it works.
+
+### Understanding Configuration Structure
+
+Let's dive deep into how configurations work in Crawl4AI. Every configuration object follows a consistent pattern of `type` and `params`. This structure enables complex, nested configurations while maintaining clarity.
+
+#### The Basic Pattern
+
+Try this in Python to understand the structure:
+```python
+from crawl4ai import BrowserConfig
+
+# Create a config and see its structure
+config = BrowserConfig(headless=True)
+print(config.dump())
+```
+
+This outputs:
+```json
+{
+    "type": "BrowserConfig",
+    "params": {
+        "headless": true
+    }
+}
+```
+
+#### Simple vs Complex Values
+
+The structure follows these rules:
+- Simple values (strings, numbers, booleans, lists) are passed directly
+- Complex values (classes, dictionaries) use the type-params pattern
+
+For example, with dictionaries:
+```json
+{
+    "browser_config": {
+        "type": "BrowserConfig",
+        "params": {
+            "headless": true,           // Simple boolean - direct value
+            "viewport": {               // Complex dictionary - needs type-params
+                "type": "dict",
+                "value": {
+                    "width": 1200,
+                    "height": 800
+                }
+            }
+        }
+    }
+}
+```
+
+#### Strategy Pattern and Nesting
+
+Strategies (like chunking or content filtering) demonstrate why we need this structure. Consider this chunking configuration:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "chunking_strategy": {
+                "type": "RegexChunking",      // Strategy implementation
+                "params": {
+                    "patterns": ["\n\n", "\\.\\s+"]
+                }
+            }
+        }
+    }
+}
+```
+
+Here, `chunking_strategy` accepts any chunking implementation. The `type` field tells the system which strategy to use, and `params` configures that specific strategy.
+
+#### Complex Nested Example
+
+Let's look at a more complex example with content filtering:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+This shows how deeply configurations can nest while maintaining a consistent structure.
+
+#### Quick Grammar Overview
+```
+config := {
+    "type": string,
+    "params": {
+        key: simple_value | complex_value
+    }
+}
+
+simple_value := string | number | boolean | [simple_value]
+complex_value := config | dict_value
+
+dict_value := {
+    "type": "dict",
+    "value": object
+}
+```
+
+#### Important Rules 🚨
+
+- Always use the type-params pattern for class instances
+- Use direct values for primitives (numbers, strings, booleans)
+- Wrap dictionaries with {"type": "dict", "value": {...}}
+- Arrays/lists are passed directly without type-params
+- All parameters are optional unless specifically required
+
+#### Pro Tip 💡
+
+The easiest way to get the correct structure is to:
+1. Create configuration objects in Python
+2. Use the `dump()` method to see their JSON representation
+3. Use that JSON in your API calls
+
+Example:
+```python
+from crawl4ai import CrawlerRunConfig, PruningContentFilter
+
+config = CrawlerRunConfig(
+    markdown_generator=DefaultMarkdownGenerator(
+        content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed")
+    ),
+    cache_mode= CacheMode.BYPASS
+)
+print(config.dump())  # Use this JSON in your API calls
+```
+
+
+#### More Examples
+
+**Advanced Crawler Configuration**
+
+```json
+{
+    "urls": ["https://example.com"],
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "cache_mode": "bypass",
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed",
+                            "min_word_threshold": 0
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+**Extraction Strategy**:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "extraction_strategy": {
+                "type": "JsonCssExtractionStrategy",
+                "params": {
+                    "schema": {
+                        "baseSelector": "article.post",
+                        "fields": [
+                            {"name": "title", "selector": "h1", "type": "text"},
+                            {"name": "content", "selector": ".content", "type": "html"}
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+**LLM Extraction Strategy**
+
+```json
+{
+  "crawler_config": {
+    "type": "CrawlerRunConfig",
+    "params": {
+      "extraction_strategy": {
+        "type": "LLMExtractionStrategy",
+        "params": {
+          "instruction": "Extract article title, author, publication date and main content",
+          "provider": "openai/gpt-4",
+          "api_token": "your-api-token",
+          "schema": {
+            "type": "dict",
+            "value": {
+              "title": "Article Schema",
+              "type": "object",
+              "properties": {
+                "title": {
+                  "type": "string",
+                  "description": "The article's headline"
+                },
+                "author": {
+                  "type": "string",
+                  "description": "The author's name"
+                },
+                "published_date": {
+                  "type": "string",
+                  "format": "date-time",
+                  "description": "Publication date and time"
+                },
+                "content": {
+                  "type": "string",
+                  "description": "The main article content"
+                }
+              },
+              "required": ["title", "content"]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+**Deep Crawler Example**
+
+```json
+{
+  "crawler_config": {
+    "type": "CrawlerRunConfig",
+    "params": {
+      "deep_crawl_strategy": {
+        "type": "BFSDeepCrawlStrategy",
+        "params": {
+          "max_depth": 3,
+          "filter_chain": {
+            "type": "FilterChain",
+            "params": {
+              "filters": [
+                {
+                  "type": "ContentTypeFilter",
+                  "params": {
+                    "allowed_types": ["text/html", "application/xhtml+xml"]
+                  }
+                },
+                {
+                  "type": "DomainFilter",
+                  "params": {
+                    "allowed_domains": ["blog.*", "docs.*"],
+                  }
+                }
+              ]
+            }
+          },
+          "url_scorer": {
+            "type": "CompositeScorer",
+            "params": {
+              "scorers": [
+                {
+                  "type": "KeywordRelevanceScorer",
+                  "params": {
+                    "keywords": ["tutorial", "guide", "documentation"],
+                  }
+                },
+                {
+                  "type": "PathDepthScorer",
+                  "params": {
+                    "weight": 0.5,
+                    "optimal_depth": 3  
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+### REST API Examples
+
+Let's look at some practical examples:
+
+#### Simple Crawl
+
+```python
+import requests
+
+crawl_payload = {
+    "urls": ["https://example.com"],
+    "browser_config": {"headless": True},
+    "crawler_config": {"stream": False}
+}
+response = requests.post(
+    "http://localhost:8000/crawl",
+    # headers={"Authorization": f"Bearer {token}"},  # If JWT is enabled, more on this later
+    json=crawl_payload
+)
+print(response.json())  # Print the response for debugging
+```
+
+#### Streaming Results
+
+```python
+async def test_stream_crawl(session, token: str):
+    """Test the /crawl/stream endpoint with multiple URLs."""
+    url = "http://localhost:8000/crawl/stream"
+    payload = {
+        "urls": [
+            "https://example.com",
+            "https://example.com/page1",  
+            "https://example.com/page2",  
+            "https://example.com/page3",  
+        ],
+        "browser_config": {"headless": True, "viewport": {"width": 1200}},
+        "crawler_config": {"stream": True, "cache_mode": "bypass"}
+    }
+
+    # headers = {"Authorization": f"Bearer {token}"} # If JWT is enabled, more on this later
+    
+    try:
+        async with session.post(url, json=payload, headers=headers) as response:
+            status = response.status
+            print(f"Status: {status} (Expected: 200)")
+            assert status == 200, f"Expected 200, got {status}"
+            
+            # Read streaming response line-by-line (NDJSON)
+            async for line in response.content:
+                if line:
+                    data = json.loads(line.decode('utf-8').strip())
+                    print(f"Streamed Result: {json.dumps(data, indent=2)}")
+    except Exception as e:
+        print(f"Error in streaming crawl test: {str(e)}")
+```
+
+## Metrics & Monitoring
+
+Keep an eye on your crawler with these endpoints:
+
+- `/health` - Quick health check
+- `/metrics` - Detailed Prometheus metrics
+- `/schema` - Full API schema
+
+Example health check:
+```bash
+curl http://localhost:8000/health
+```
+
+## Deployment Scenarios
+
+> 🚧 Coming soon! We'll cover:
+> - Kubernetes deployment
+> - Cloud provider setups (AWS, GCP, Azure)
+> - High-availability configurations
+> - Load balancing strategies
+
+## Complete Examples
+
+Check out the `examples` folder in our repository for full working examples! Here are two to get you started:
+[Using Client SDK](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_sdk.py)
+[Using REST API](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_rest_api.py)
+
+## Server Configuration
+
+The server's behavior can be customized through the `config.yml` file. Let's explore how to configure your Crawl4AI server for optimal performance and security.
+
+### Understanding config.yml
+
+The configuration file is located at `deploy/docker/config.yml`. You can either modify this file before building the image or mount a custom configuration when running the container.
+
+Here's a detailed breakdown of the configuration options:
+
+```yaml
+# Application Configuration
+app:
+  title: "Crawl4AI API"           # Server title in OpenAPI docs
+  version: "1.0.0"               # API version
+  host: "0.0.0.0"               # Listen on all interfaces
+  port: 8000                    # Server port
+  reload: True                  # Enable hot reloading (development only)
+  timeout_keep_alive: 300       # Keep-alive timeout in seconds
+
+# Rate Limiting Configuration
+rate_limiting:
+  enabled: True                 # Enable/disable rate limiting
+  default_limit: "100/minute"   # Rate limit format: "number/timeunit"
+  trusted_proxies: []          # List of trusted proxy IPs
+  storage_uri: "memory://"     # Use "redis://localhost:6379" for production
+
+# Security Configuration
+security:
+  enabled: false               # Master toggle for security features
+  jwt_enabled: true            # Enable JWT authentication
+  https_redirect: True         # Force HTTPS
+  trusted_hosts: ["*"]         # Allowed hosts (use specific domains in production)
+  headers:                     # Security headers
+    x_content_type_options: "nosniff"
+    x_frame_options: "DENY"
+    content_security_policy: "default-src 'self'"
+    strict_transport_security: "max-age=63072000; includeSubDomains"
+
+# Crawler Configuration
+crawler:
+  memory_threshold_percent: 95.0  # Memory usage threshold
+  rate_limiter:
+    base_delay: [1.0, 2.0]      # Min and max delay between requests
+  timeouts:
+    stream_init: 30.0           # Stream initialization timeout
+    batch_process: 300.0        # Batch processing timeout
+
+# Logging Configuration
+logging:
+  level: "INFO"                 # Log level (DEBUG, INFO, WARNING, ERROR)
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Observability Configuration
+observability:
+  prometheus:
+    enabled: True              # Enable Prometheus metrics
+    endpoint: "/metrics"       # Metrics endpoint
+  health_check:
+    endpoint: "/health"        # Health check endpoint
+```
+
+### JWT Authentication
+
+When `security.jwt_enabled` is set to `true` in your config.yml, all endpoints require JWT authentication via bearer tokens. Here's how it works:
+
+#### Getting a Token
+```python
+POST /token
+Content-Type: application/json
+
+{
+    "email": "user@example.com"
+}
+```
+
+The endpoint returns:
+```json
+{
+    "email": "user@example.com",
+    "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOi...",
+    "token_type": "bearer"
+}
+```
+
+#### Using the Token
+Add the token to your requests:
+```bash
+curl -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGci..." http://localhost:8000/crawl
+```
+
+Using the Python SDK:
+```python
+from crawl4ai.docker_client import Crawl4aiDockerClient
+
+async with Crawl4aiDockerClient() as client:
+    # Authenticate first
+    await client.authenticate("user@example.com")
+    
+    # Now all requests will include the token automatically
+    result = await client.crawl(urls=["https://example.com"])
+```
+
+#### Production Considerations 💡
+The default implementation uses a simple email verification. For production use, consider:
+- Email verification via OTP/magic links
+- OAuth2 integration
+- Rate limiting token generation
+- Token expiration and refresh mechanisms
+- IP-based restrictions
+
+### Configuration Tips and Best Practices
+
+1. **Production Settings** 🏭
+
+   ```yaml
+   app:
+     reload: False              # Disable reload in production
+     timeout_keep_alive: 120    # Lower timeout for better resource management
+   
+   rate_limiting:
+     storage_uri: "redis://redis:6379"  # Use Redis for distributed rate limiting
+     default_limit: "50/minute"         # More conservative rate limit
+   
+   security:
+     enabled: true                      # Enable all security features
+     trusted_hosts: ["your-domain.com"] # Restrict to your domain
+   ```
+
+2. **Development Settings** 🛠️
+
+   ```yaml
+   app:
+     reload: True               # Enable hot reloading
+     timeout_keep_alive: 300    # Longer timeout for debugging
+   
+   logging:
+     level: "DEBUG"            # More verbose logging
+   ```
+
+3. **High-Traffic Settings** 🚦
+
+   ```yaml
+   crawler:
+     memory_threshold_percent: 85.0  # More conservative memory limit
+     rate_limiter:
+       base_delay: [2.0, 4.0]       # More aggressive rate limiting
+   ```
+
+### Customizing Your Configuration
+
+#### Method 1: Pre-build Configuration
+
+```bash
+# Copy and modify config before building
+cd crawl4ai/deploy
+vim custom-config.yml # Or use any editor
+
+# Build with custom config
+docker build --platform=linux/amd64 --no-cache -t crawl4ai:latest .
+```
+
+#### Method 2: Build-time Configuration
+
+Use a custom config during build:
+
+```bash
+# Build with custom config
+docker build --platform=linux/amd64 --no-cache \
+  --build-arg CONFIG_PATH=/path/to/custom-config.yml \ 
+  -t crawl4ai:latest .
+```
+
+#### Method 3: Runtime Configuration
+```bash
+# Mount custom config at runtime
+docker run -d -p 8000:8000 \
+  -v $(pwd)/custom-config.yml:/app/config.yml \
+  crawl4ai-server:prod
+```
+
+> 💡 Note: When using Method 2, `/path/to/custom-config.yml` is relative to deploy directory.
+> 💡 Note: When using Method 3, ensure your custom config file has all required fields as the container will use this instead of the built-in config.
+
+### Configuration Recommendations
+
+1. **Security First** 🔒
+   - Always enable security in production
+   - Use specific trusted_hosts instead of wildcards
+   - Set up proper rate limiting to protect your server
+   - Consider your environment before enabling HTTPS redirect
+
+2. **Resource Management** 💻
+   - Adjust memory_threshold_percent based on available RAM
+   - Set timeouts according to your content size and network conditions
+   - Use Redis for rate limiting in multi-container setups
+
+3. **Monitoring** 📊
+   - Enable Prometheus if you need metrics
+   - Set DEBUG logging in development, INFO in production
+   - Regular health check monitoring is crucial
+
+4. **Performance Tuning** ⚡
+   - Start with conservative rate limiter delays
+   - Increase batch_process timeout for large content
+   - Adjust stream_init timeout based on initial response times
+
+## Getting Help
+
+We're here to help you succeed with Crawl4AI! Here's how to get support:
+
+- 📖 Check our [full documentation](https://docs.crawl4ai.com)
+- 🐛 Found a bug? [Open an issue](https://github.com/unclecode/crawl4ai/issues)
+- 💬 Join our [Discord community](https://discord.gg/crawl4ai)
+- ⭐ Star us on GitHub to show support!
+
+## Summary
+
+In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment:
+- Building and running the Docker container
+- Configuring the environment
+- Making API requests with proper typing
+- Using the Python SDK
+- Monitoring your deployment
+
+Remember, the examples in the `examples` folder are your friends - they show real-world usage patterns that you can adapt for your needs.
+
+Keep exploring, and don't hesitate to reach out if you need help! We're building something amazing together. 🚀
+
+Happy crawling! 🕷️

deploy/docker/api.py

@@ -0,0 +1,449 @@
+import os
+import json
+import asyncio
+from typing import List, Tuple
+from functools import partial
+
+import logging
+from typing import Optional, AsyncGenerator
+from urllib.parse import unquote
+from fastapi import HTTPException, Request, status
+from fastapi.background import BackgroundTasks
+from fastapi.responses import JSONResponse
+from redis import asyncio as aioredis
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    LLMExtractionStrategy,
+    CacheMode,
+    BrowserConfig,
+    MemoryAdaptiveDispatcher,
+    RateLimiter, 
+    LLMConfig
+)
+from crawl4ai.utils import perform_completion_with_backoff
+from crawl4ai.content_filter_strategy import (
+    PruningContentFilter,
+    BM25ContentFilter,
+    LLMContentFilter
+)
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+
+from utils import (
+    TaskStatus,
+    FilterType,
+    get_base_url,
+    is_task_id,
+    should_cleanup_task,
+    decode_redis_hash
+)
+
+logger = logging.getLogger(__name__)
+
+async def handle_llm_qa(
+    url: str,
+    query: str,
+    config: dict
+) -> str:
+    """Process QA using LLM with crawled content as context."""
+    try:
+        # Extract base URL by finding last '?q=' occurrence
+        last_q_index = url.rfind('?q=')
+        if last_q_index != -1:
+            url = url[:last_q_index]
+
+        # Get markdown content
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(url)
+            if not result.success:
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=result.error_message
+                )
+            content = result.markdown.fit_markdown
+
+        # Create prompt and get LLM response
+        prompt = f"""Use the following content as context to answer the question.
+    Content:
+    {content}
+
+    Question: {query}
+
+    Answer:"""
+
+        response = perform_completion_with_backoff(
+            provider=config["llm"]["provider"],
+            prompt_with_variables=prompt,
+            api_token=os.environ.get(config["llm"].get("api_key_env", ""))
+        )
+
+        return response.choices[0].message.content
+    except Exception as e:
+        logger.error(f"QA processing error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def process_llm_extraction(
+    redis: aioredis.Redis,
+    config: dict,
+    task_id: str,
+    url: str,
+    instruction: str,
+    schema: Optional[str] = None,
+    cache: str = "0"
+) -> None:
+    """Process LLM extraction in background."""
+    try:
+        # If config['llm'] has api_key then ignore the api_key_env
+        api_key = ""
+        if "api_key" in config["llm"]:
+            api_key = config["llm"]["api_key"]
+        else:
+            api_key = os.environ.get(config["llm"].get("api_key_env", None), "")
+        llm_strategy = LLMExtractionStrategy(
+            llm_config=LLMConfig(
+                provider=config["llm"]["provider"],
+                api_token=api_key
+            ),
+            instruction=instruction,
+            schema=json.loads(schema) if schema else None,
+        )
+
+        cache_mode = CacheMode.ENABLED if cache == "1" else CacheMode.WRITE_ONLY
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(
+                    extraction_strategy=llm_strategy,
+                    scraping_strategy=LXMLWebScrapingStrategy(),
+                    cache_mode=cache_mode
+                )
+            )
+
+        if not result.success:
+            await redis.hset(f"task:{task_id}", mapping={
+                "status": TaskStatus.FAILED,
+                "error": result.error_message
+            })
+            return
+
+        try:
+            content = json.loads(result.extracted_content)
+        except json.JSONDecodeError:
+            content = result.extracted_content
+        await redis.hset(f"task:{task_id}", mapping={
+            "status": TaskStatus.COMPLETED,
+            "result": json.dumps(content)
+        })
+
+    except Exception as e:
+        logger.error(f"LLM extraction error: {str(e)}", exc_info=True)
+        await redis.hset(f"task:{task_id}", mapping={
+            "status": TaskStatus.FAILED,
+            "error": str(e)
+        })
+
+async def handle_markdown_request(
+    url: str,
+    filter_type: FilterType,
+    query: Optional[str] = None,
+    cache: str = "0",
+    config: Optional[dict] = None
+) -> str:
+    """Handle markdown generation requests."""
+    try:
+        decoded_url = unquote(url)
+        if not decoded_url.startswith(('http://', 'https://')):
+            decoded_url = 'https://' + decoded_url
+
+        if filter_type == FilterType.RAW:
+            md_generator = DefaultMarkdownGenerator()
+        else:
+            content_filter = {
+                FilterType.FIT: PruningContentFilter(),
+                FilterType.BM25: BM25ContentFilter(user_query=query or ""),
+                FilterType.LLM: LLMContentFilter(
+                    llm_config=LLMConfig(
+                        provider=config["llm"]["provider"],
+                        api_token=os.environ.get(config["llm"].get("api_key_env", None), ""),
+                    ),
+                    instruction=query or "Extract main content"
+                )
+            }[filter_type]
+            md_generator = DefaultMarkdownGenerator(content_filter=content_filter)
+
+        cache_mode = CacheMode.ENABLED if cache == "1" else CacheMode.WRITE_ONLY
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=decoded_url,
+                config=CrawlerRunConfig(
+                    markdown_generator=md_generator,
+                    scraping_strategy=LXMLWebScrapingStrategy(),
+                    cache_mode=cache_mode
+                )
+            )
+            
+            if not result.success:
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=result.error_message
+                )
+
+            return (result.markdown.raw_markdown 
+                   if filter_type == FilterType.RAW 
+                   else result.markdown.fit_markdown)
+
+    except Exception as e:
+        logger.error(f"Markdown error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def handle_llm_request(
+    redis: aioredis.Redis,
+    background_tasks: BackgroundTasks,
+    request: Request,
+    input_path: str,
+    query: Optional[str] = None,
+    schema: Optional[str] = None,
+    cache: str = "0",
+    config: Optional[dict] = None
+) -> JSONResponse:
+    """Handle LLM extraction requests."""
+    base_url = get_base_url(request)
+    
+    try:
+        if is_task_id(input_path):
+            return await handle_task_status(
+                redis, input_path, base_url
+            )
+
+        if not query:
+            return JSONResponse({
+                "message": "Please provide an instruction",
+                "_links": {
+                    "example": {
+                        "href": f"{base_url}/llm/{input_path}?q=Extract+main+content",
+                        "title": "Try this example"
+                    }
+                }
+            })
+
+        return await create_new_task(
+            redis,
+            background_tasks,
+            input_path,
+            query,
+            schema,
+            cache,
+            base_url,
+            config
+        )
+
+    except Exception as e:
+        logger.error(f"LLM endpoint error: {str(e)}", exc_info=True)
+        return JSONResponse({
+            "error": str(e),
+            "_links": {
+                "retry": {"href": str(request.url)}
+            }
+        }, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+async def handle_task_status(
+    redis: aioredis.Redis,
+    task_id: str,
+    base_url: str
+) -> JSONResponse:
+    """Handle task status check requests."""
+    task = await redis.hgetall(f"task:{task_id}")
+    if not task:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Task not found"
+        )
+
+    task = decode_redis_hash(task)
+    response = create_task_response(task, task_id, base_url)
+
+    if task["status"] in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
+        if should_cleanup_task(task["created_at"]):
+            await redis.delete(f"task:{task_id}")
+
+    return JSONResponse(response)
+
+async def create_new_task(
+    redis: aioredis.Redis,
+    background_tasks: BackgroundTasks,
+    input_path: str,
+    query: str,
+    schema: Optional[str],
+    cache: str,
+    base_url: str,
+    config: dict
+) -> JSONResponse:
+    """Create and initialize a new task."""
+    decoded_url = unquote(input_path)
+    if not decoded_url.startswith(('http://', 'https://')):
+        decoded_url = 'https://' + decoded_url
+
+    from datetime import datetime
+    task_id = f"llm_{int(datetime.now().timestamp())}_{id(background_tasks)}"
+    
+    await redis.hset(f"task:{task_id}", mapping={
+        "status": TaskStatus.PROCESSING,
+        "created_at": datetime.now().isoformat(),
+        "url": decoded_url
+    })
+
+    background_tasks.add_task(
+        process_llm_extraction,
+        redis,
+        config,
+        task_id,
+        decoded_url,
+        query,
+        schema,
+        cache
+    )
+
+    return JSONResponse({
+        "task_id": task_id,
+        "status": TaskStatus.PROCESSING,
+        "url": decoded_url,
+        "_links": {
+            "self": {"href": f"{base_url}/llm/{task_id}"},
+            "status": {"href": f"{base_url}/llm/{task_id}"}
+        }
+    })
+
+def create_task_response(task: dict, task_id: str, base_url: str) -> dict:
+    """Create response for task status check."""
+    response = {
+        "task_id": task_id,
+        "status": task["status"],
+        "created_at": task["created_at"],
+        "url": task["url"],
+        "_links": {
+            "self": {"href": f"{base_url}/llm/{task_id}"},
+            "refresh": {"href": f"{base_url}/llm/{task_id}"}
+        }
+    }
+
+    if task["status"] == TaskStatus.COMPLETED:
+        response["result"] = json.loads(task["result"])
+    elif task["status"] == TaskStatus.FAILED:
+        response["error"] = task["error"]
+
+    return response
+
+async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) -> AsyncGenerator[bytes, None]:
+    """Stream results with heartbeats and completion markers."""
+    import json
+    from utils import datetime_handler
+
+    try:
+        async for result in results_gen:
+            try:
+                result_dict = result.model_dump()
+                logger.info(f"Streaming result for {result_dict.get('url', 'unknown')}")
+                data = json.dumps(result_dict, default=datetime_handler) + "\n"
+                yield data.encode('utf-8')
+            except Exception as e:
+                logger.error(f"Serialization error: {e}")
+                error_response = {"error": str(e), "url": getattr(result, 'url', 'unknown')}
+                yield (json.dumps(error_response) + "\n").encode('utf-8')
+
+        yield json.dumps({"status": "completed"}).encode('utf-8')
+        
+    except asyncio.CancelledError:
+        logger.warning("Client disconnected during streaming")
+    finally:
+        try:
+            await crawler.close()
+        except Exception as e:
+            logger.error(f"Crawler cleanup error: {e}")
+
+async def handle_crawl_request(
+    urls: List[str],
+    browser_config: dict,
+    crawler_config: dict,
+    config: dict
+) -> dict:
+    """Handle non-streaming crawl requests."""
+    try:
+        browser_config = BrowserConfig.load(browser_config)
+        crawler_config = CrawlerRunConfig.load(crawler_config)
+
+        dispatcher = MemoryAdaptiveDispatcher(
+            memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
+            rate_limiter=RateLimiter(
+                base_delay=tuple(config["crawler"]["rate_limiter"]["base_delay"])
+            )
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            results = []
+            func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
+            partial_func = partial(func, 
+                                   urls[0] if len(urls) == 1 else urls, 
+                                   config=crawler_config, 
+                                   dispatcher=dispatcher)
+            results = await partial_func()
+            return {
+                "success": True,
+                "results": [result.model_dump() for result in results]
+            }
+
+    except Exception as e:
+        logger.error(f"Crawl error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def handle_stream_crawl_request(
+    urls: List[str],
+    browser_config: dict,
+    crawler_config: dict,
+    config: dict
+) -> Tuple[AsyncWebCrawler, AsyncGenerator]:
+    """Handle streaming crawl requests."""
+    try:
+        browser_config = BrowserConfig.load(browser_config)
+        browser_config.verbose = True
+        crawler_config = CrawlerRunConfig.load(crawler_config)
+        crawler_config.scraping_strategy = LXMLWebScrapingStrategy()
+
+        dispatcher = MemoryAdaptiveDispatcher(
+            memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
+            rate_limiter=RateLimiter(
+                base_delay=tuple(config["crawler"]["rate_limiter"]["base_delay"])
+            )
+        )
+
+        crawler = AsyncWebCrawler(config=browser_config)
+        await crawler.start()
+
+        results_gen = await crawler.arun_many(
+            urls=urls,
+            config=crawler_config,
+            dispatcher=dispatcher
+        )
+
+        return crawler, results_gen
+
+    except Exception as e:
+        if 'crawler' in locals():
+            await crawler.close()
+        logger.error(f"Stream crawl error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )

deploy/docker/auth.py

@@ -0,0 +1,55 @@
+import os
+from datetime import datetime, timedelta, timezone
+from typing import Dict, Optional
+from jwt import JWT, jwk_from_dict
+from jwt.utils import get_int_from_datetime
+from fastapi import Depends, HTTPException
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from pydantic import EmailStr
+from pydantic.main import BaseModel
+import base64
+
+instance = JWT()
+security = HTTPBearer(auto_error=False)
+SECRET_KEY = os.environ.get("SECRET_KEY", "mysecret")
+ACCESS_TOKEN_EXPIRE_MINUTES = 60
+
+def get_jwk_from_secret(secret: str):
+    """Convert a secret string into a JWK object."""
+    secret_bytes = secret.encode('utf-8')
+    b64_secret = base64.urlsafe_b64encode(secret_bytes).rstrip(b'=').decode('utf-8')
+    return jwk_from_dict({"kty": "oct", "k": b64_secret})
+
+def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str:
+    """Create a JWT access token with an expiration."""
+    to_encode = data.copy()
+    expire = datetime.now(timezone.utc) + (expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES))
+    to_encode.update({"exp": get_int_from_datetime(expire)})
+    signing_key = get_jwk_from_secret(SECRET_KEY)
+    return instance.encode(to_encode, signing_key, alg='HS256')
+
+def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)) -> Dict:
+    """Verify the JWT token from the Authorization header."""
+
+    if credentials is None:
+        return None
+    token = credentials.credentials
+    verifying_key = get_jwk_from_secret(SECRET_KEY)
+    try:
+        payload = instance.decode(token, verifying_key, do_time_check=True, algorithms='HS256')
+        return payload
+    except Exception:
+        raise HTTPException(status_code=401, detail="Invalid or expired token")
+
+
+def get_token_dependency(config: Dict):
+    """Return the token dependency if JWT is enabled, else a function that returns None."""
+
+    if config.get("security", {}).get("jwt_enabled", False):
+        return verify_token
+    else:
+        return lambda: None
+
+
+class TokenRequest(BaseModel):
+    email: EmailStr

deploy/docker/config.yml

@@ -0,0 +1,71 @@
+# Application Configuration
+app:
+  title: "Crawl4AI API"
+  version: "1.0.0"
+  host: "0.0.0.0"
+  port: 8020
+  reload: True
+  timeout_keep_alive: 300
+
+# Default LLM Configuration
+llm:
+  provider: "openai/gpt-4o-mini"
+  api_key_env: "OPENAI_API_KEY"
+  # api_key: sk-...  # If you pass the API key directly then api_key_env will be ignored
+
+# Redis Configuration
+redis:
+  host: "localhost"
+  port: 6379
+  db: 0
+  password: ""
+  ssl: False
+  ssl_cert_reqs: None
+  ssl_ca_certs: None
+  ssl_certfile: None
+  ssl_keyfile: None
+  ssl_cert_reqs: None
+  ssl_ca_certs: None
+  ssl_certfile: None
+  ssl_keyfile: None
+
+# Rate Limiting Configuration
+rate_limiting:
+  enabled: True
+  default_limit: "1000/minute"
+  trusted_proxies: []
+  storage_uri: "memory://"  # Use "redis://localhost:6379" for production
+
+# Security Configuration
+security:
+  enabled: false 
+  jwt_enabled: false 
+  https_redirect: false
+  trusted_hosts: ["*"]
+  headers:
+    x_content_type_options: "nosniff"
+    x_frame_options: "DENY"
+    content_security_policy: "default-src 'self'"
+    strict_transport_security: "max-age=63072000; includeSubDomains"
+
+# Crawler Configuration
+crawler:
+  memory_threshold_percent: 95.0
+  rate_limiter:
+    base_delay: [1.0, 2.0]
+  timeouts:
+    stream_init: 30.0  # Timeout for stream initialization
+    batch_process: 300.0  # Timeout for batch processing
+
+# Logging Configuration
+logging:
+  level: "INFO"
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Observability Configuration
+observability:
+  prometheus:
+    enabled: True
+    endpoint: "/metrics"
+  health_check:
+    endpoint: "/health"

deploy/docker/requirements.txt

@@ -0,0 +1,10 @@
+crawl4ai
+fastapi
+uvicorn
+gunicorn>=23.0.0
+slowapi>=0.1.9
+prometheus-fastapi-instrumentator>=7.0.2
+redis>=5.2.1
+jwt>=1.3.1
+dnspython>=2.7.0
+email-validator>=2.2.0

deploy/docker/server.py

@@ -0,0 +1,181 @@
+import os
+import sys
+import time
+from typing import List, Optional, Dict
+from fastapi import FastAPI, HTTPException, Request, Query, Path, Depends
+from fastapi.responses import StreamingResponse, RedirectResponse, PlainTextResponse, JSONResponse
+from fastapi.middleware.httpsredirect import HTTPSRedirectMiddleware
+from fastapi.middleware.trustedhost import TrustedHostMiddleware
+from pydantic import BaseModel, Field
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+from prometheus_fastapi_instrumentator import Instrumentator
+from redis import asyncio as aioredis
+
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+from utils import FilterType, load_config, setup_logging, verify_email_domain
+from api import (
+    handle_markdown_request,
+    handle_llm_qa,
+    handle_stream_crawl_request,
+    handle_crawl_request,
+    stream_results
+)
+from auth import create_access_token, get_token_dependency, TokenRequest  # Import from auth.py
+
+__version__ = "0.2.6"
+
+class CrawlRequest(BaseModel):
+    urls: List[str] = Field(min_length=1, max_length=100)
+    browser_config: Optional[Dict] = Field(default_factory=dict)
+    crawler_config: Optional[Dict] = Field(default_factory=dict)
+
+# Load configuration and setup
+config = load_config()
+setup_logging(config)
+
+# Initialize Redis
+redis = aioredis.from_url(config["redis"].get("uri", "redis://localhost"))
+
+# Initialize rate limiter
+limiter = Limiter(
+    key_func=get_remote_address,
+    default_limits=[config["rate_limiting"]["default_limit"]],
+    storage_uri=config["rate_limiting"]["storage_uri"]
+)
+
+app = FastAPI(
+    title=config["app"]["title"],
+    version=config["app"]["version"]
+)
+
+# Configure middleware
+def setup_security_middleware(app, config):
+    sec_config = config.get("security", {})
+    if sec_config.get("enabled", False):
+        if sec_config.get("https_redirect", False):
+            app.add_middleware(HTTPSRedirectMiddleware)
+        if sec_config.get("trusted_hosts", []) != ["*"]:
+            app.add_middleware(TrustedHostMiddleware, allowed_hosts=sec_config["trusted_hosts"])
+
+setup_security_middleware(app, config)
+
+# Prometheus instrumentation
+if config["observability"]["prometheus"]["enabled"]:
+    Instrumentator().instrument(app).expose(app)
+
+# Get token dependency based on config
+token_dependency = get_token_dependency(config)
+
+# Middleware for security headers
+@app.middleware("http")
+async def add_security_headers(request: Request, call_next):
+    response = await call_next(request)
+    if config["security"]["enabled"]:
+        response.headers.update(config["security"]["headers"])
+    return response
+
+# Token endpoint (always available, but usage depends on config)
+@app.post("/token")
+async def get_token(request_data: TokenRequest):
+    if not verify_email_domain(request_data.email):
+        raise HTTPException(status_code=400, detail="Invalid email domain")
+    token = create_access_token({"sub": request_data.email})
+    return {"email": request_data.email, "access_token": token, "token_type": "bearer"}
+
+# Endpoints with conditional auth
+@app.get("/md/{url:path}")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def get_markdown(
+    request: Request,
+    url: str,
+    f: FilterType = FilterType.FIT,
+    q: Optional[str] = None,
+    c: Optional[str] = "0",
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    result = await handle_markdown_request(url, f, q, c, config)
+    return PlainTextResponse(result)
+
+@app.get("/llm/{url:path}", description="URL should be without http/https prefix")
+async def llm_endpoint(
+    request: Request,
+    url: str = Path(...),
+    q: Optional[str] = Query(None),
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not q:
+        raise HTTPException(status_code=400, detail="Query parameter 'q' is required")
+    if not url.startswith(('http://', 'https://')):
+        url = 'https://' + url
+    try:
+        answer = await handle_llm_qa(url, q, config)
+        return JSONResponse({"answer": answer})
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/schema")
+async def get_schema():
+    from crawl4ai import BrowserConfig, CrawlerRunConfig
+    return {"browser": BrowserConfig().dump(), "crawler": CrawlerRunConfig().dump()}
+
+@app.get(config["observability"]["health_check"]["endpoint"])
+async def health():
+    return {"status": "ok", "timestamp": time.time(), "version": __version__}
+
+@app.get(config["observability"]["prometheus"]["endpoint"])
+async def metrics():
+    return RedirectResponse(url=config["observability"]["prometheus"]["endpoint"])
+
+@app.post("/crawl")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def crawl(
+    request: Request,
+    crawl_request: CrawlRequest,
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not crawl_request.urls:
+        raise HTTPException(status_code=400, detail="At least one URL required")
+    
+    results = await handle_crawl_request(
+        urls=crawl_request.urls,
+        browser_config=crawl_request.browser_config,
+        crawler_config=crawl_request.crawler_config,
+        config=config
+    )
+
+    return JSONResponse(results)
+
+
+@app.post("/crawl/stream")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def crawl_stream(
+    request: Request,
+    crawl_request: CrawlRequest,
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not crawl_request.urls:
+        raise HTTPException(status_code=400, detail="At least one URL required")
+
+    crawler, results_gen = await handle_stream_crawl_request(
+        urls=crawl_request.urls,
+        browser_config=crawl_request.browser_config,
+        crawler_config=crawl_request.crawler_config,
+        config=config
+    )
+
+    return StreamingResponse(
+        stream_results(crawler, results_gen),
+        media_type='application/x-ndjson',
+        headers={'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'X-Stream-Status': 'active'}
+    )
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(
+        "server:app",
+        host=config["app"]["host"],
+        port=config["app"]["port"],
+        reload=config["app"]["reload"],
+        timeout_keep_alive=config["app"]["timeout_keep_alive"]
+    )

deploy/docker/supervisord.conf

@@ -0,0 +1,12 @@
+[supervisord]
+nodaemon=true
+
+[program:redis]
+command=redis-server
+autorestart=true
+priority=10
+
+[program:gunicorn]
+command=gunicorn --bind 0.0.0.0:8000 --workers 4 --threads 2 --timeout 300 --graceful-timeout 60 --keep-alive 65 --log-level debug --worker-class uvicorn.workers.UvicornWorker --max-requests 1000 --max-requests-jitter 50 server:app
+autorestart=true
+priority=20

deploy/docker/utils.py

@@ -0,0 +1,66 @@
+import dns.resolver
+import logging
+import yaml
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from fastapi import Request
+from typing import Dict, Optional
+
+class TaskStatus(str, Enum):
+    PROCESSING = "processing"
+    FAILED = "failed"
+    COMPLETED = "completed"
+
+class FilterType(str, Enum):
+    RAW = "raw"
+    FIT = "fit"
+    BM25 = "bm25"
+    LLM = "llm"
+
+def load_config() -> Dict:
+    """Load and return application configuration."""
+    config_path = Path(__file__).parent / "config.yml"
+    with open(config_path, "r") as config_file:
+        return yaml.safe_load(config_file)
+
+def setup_logging(config: Dict) -> None:
+    """Configure application logging."""
+    logging.basicConfig(
+        level=config["logging"]["level"],
+        format=config["logging"]["format"]
+    )
+
+def get_base_url(request: Request) -> str:
+    """Get base URL including scheme and host."""
+    return f"{request.url.scheme}://{request.url.netloc}"
+
+def is_task_id(value: str) -> bool:
+    """Check if the value matches task ID pattern."""
+    return value.startswith("llm_") and "_" in value
+
+def datetime_handler(obj: any) -> Optional[str]:
+    """Handle datetime serialization for JSON."""
+    if hasattr(obj, 'isoformat'):
+        return obj.isoformat()
+    raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+def should_cleanup_task(created_at: str) -> bool:
+    """Check if task should be cleaned up based on creation time."""
+    created = datetime.fromisoformat(created_at)
+    return (datetime.now() - created).total_seconds() > 3600
+
+def decode_redis_hash(hash_data: Dict[bytes, bytes]) -> Dict[str, str]:
+    """Decode Redis hash data from bytes to strings."""
+    return {k.decode('utf-8'): v.decode('utf-8') for k, v in hash_data.items()}
+
+
+
+def verify_email_domain(email: str) -> bool:
+    try:
+        domain = email.split('@')[1]
+        # Try to resolve MX records for the domain.
+        records = dns.resolver.resolve(domain, 'MX')
+        return True if records else False
+    except Exception as e:
+        return False

docker-compose.yml

@@ -1,3 +1,30 @@
+# Base configuration (not a service, just a reusable config block)
+x-base-config: &base-config
+  ports:
+    - "11235:11235"
+    - "8000:8000"
+    - "9222:9222"
+    - "8080:8080"
+  environment:
+    - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
+    - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+    - CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
+  volumes:
+    - /dev/shm:/dev/shm
+  deploy:
+    resources:
+      limits:
+        memory: 4G
+      reservations:
+        memory: 1G
+  restart: unless-stopped
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
+    interval: 30s
+    timeout: 10s
+    retries: 3
+    start_period: 40s
+
 services:
   # Local build services for different platforms
   crawl4ai-amd64:
@@ -11,9 +38,7 @@ services:
       platforms:
         - linux/amd64
     profiles: ["local-amd64"]
-    extends: &base-config
-      file: docker-compose.yml
-      service: base-config
+    <<: *base-config  # extends yerine doğrudan yapılandırmayı dahil ettik
 
   crawl4ai-arm64:
     build:
@@ -26,42 +51,15 @@ services:
       platforms:
         - linux/arm64
     profiles: ["local-arm64"]
-    extends: *base-config
+    <<: *base-config
 
   # Hub services for different platforms and versions
   crawl4ai-hub-amd64:
     image: unclecode/crawl4ai:${VERSION:-basic}-amd64
     profiles: ["hub-amd64"]
-    extends: *base-config
+    <<: *base-config
 
   crawl4ai-hub-arm64:
     image: unclecode/crawl4ai:${VERSION:-basic}-arm64
     profiles: ["hub-arm64"]
-    extends: *base-config
-
-  # Base configuration to be extended
-  base-config:
-    ports:
-      - "11235:11235"
-      - "8000:8000"
-      - "9222:9222"
-      - "8080:8080"
-    environment:
-      - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
-      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
-      - CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
-    volumes:
-      - /dev/shm:/dev/shm
-    deploy:
-      resources:
-        limits:
-          memory: 4G
-        reservations:
-          memory: 1G
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 40s
+    <<: *base-config