fix(docs): correct section numbering in deepcrawl_example.py tutorial

Make 'crawl' the default command when no command is specified.
This improves user experience by allowing direct URL input without
explicitly specifying the 'crawl' command.

Also removes unnecessary blank lines in example code for better readability.

.do/app.yaml

@@ -1,19 +0,0 @@
-alerts:
-- rule: DEPLOYMENT_FAILED
-- rule: DOMAIN_FAILED
-name: crawl4ai
-region: nyc
-services:
-- dockerfile_path: Dockerfile
-  github:
-    branch: 0.3.74
-    deploy_on_push: true
-    repo: unclecode/crawl4ai 
-  health_check:
-    http_path: /health
-  http_port: 11235
-  instance_count: 1
-  instance_size_slug: professional-xs
-  name: web
-  routes:
-  - path: /

.do/deploy.template.yaml

@@ -1,22 +0,0 @@
-spec:
-  name: crawl4ai
-  services:
-    - name: crawl4ai
-      git:
-        branch: 0.3.74
-        repo_clone_url: https://github.com/unclecode/crawl4ai.git
-      dockerfile_path: Dockerfile
-      http_port: 11235
-      instance_count: 1
-      instance_size_slug: professional-xs
-      health_check:
-        http_path: /health
-      envs:
-        - key: INSTALL_TYPE
-          value: "basic"
-        - key: PYTHON_VERSION  
-          value: "3.10"
-        - key: ENABLE_GPU
-          value: "false"
-      routes:
-        - path: /

.gitattributes

@@ -0,0 +1,12 @@
+# Documentation
+*.html linguist-documentation
+docs/* linguist-documentation
+docs/examples/* linguist-documentation
+docs/md_v2/* linguist-documentation
+
+# Explicitly mark Python as the main language
+*.py linguist-detectable=true
+*.py linguist-language=Python
+
+# Exclude HTML from language statistics
+*.html linguist-detectable=false

.github/DISCUSSION_TEMPLATE/feature-requests.yml

@@ -0,0 +1,59 @@
+title: "[Feature Request]: "
+labels: ["⚙️ New"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for your interest in suggesting a new feature! Before you submit, please take a moment to check if already exists in
+        this discussions category to avoid duplicates. 😊
+
+  - type: textarea
+    id: needs_to_be_done
+    attributes:
+      label: What needs to be done?
+      description: Please describe the feature or functionality you'd like to see.
+      placeholder: "e.g., Return alt text along with images scraped from a webpages in Result"
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem_to_solve
+    attributes:
+      label: What problem does this solve?
+      description: Explain the pain point or issue this feature will help address.
+      placeholder: "e.g., Bypass Captchas added by cloudflare"
+    validations:
+      required: true
+
+  - type: textarea
+    id: target_users
+    attributes:
+      label: Target users/beneficiaries
+      description: Who would benefit from this feature? (e.g., specific teams, developers, users, etc.)
+      placeholder: "e.g., Marketing teams, developers"
+    validations:
+      required: false
+
+  - type: textarea
+    id: current_workarounds
+    attributes:
+      label: Current alternatives/workarounds
+      description: Are there any existing solutions or workarounds? How does this feature improve upon them?
+      placeholder: "e.g., Users manually select the css classes mapped to data fields to extract them"
+    validations:
+      required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        ### 💡 Implementation Ideas
+
+  - type: textarea
+    id: proposed_approach
+    attributes:
+      label: Proposed approach
+      description: Share any ideas you have for how this feature could be implemented. Point out any challenges your foresee
+       and the success metrics for this feature
+      placeholder: "e.g., Implement a breadth first traversal algorithm for scraper"
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,127 @@
+name: Bug Report
+description: Report a bug with the Crawl4AI.
+title: "[Bug]: "
+labels: ["🐞 Bug","🩺 Needs Triage"]
+body:
+  - type: input
+    id: crawl4ai_version
+    attributes:
+      label: crawl4ai version
+      description: Specify the version of crawl4ai you are using.
+      placeholder: "e.g., 2.0.0"
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected_behavior
+    attributes:
+      label: Expected Behavior
+      description: Describe what you expected to happen.
+      placeholder: "Provide a detailed explanation of the expected outcome."
+    validations:
+      required: true
+
+  - type: textarea
+    id: current_behavior
+    attributes:
+      label: Current Behavior
+      description: Describe what is happening instead of the expected behavior.
+      placeholder: "Describe the actual result or issue you encountered."
+    validations:
+      required: true
+
+  - type: dropdown
+    id: reproducible
+    attributes:
+      label: Is this reproducible?
+      description: Indicate whether this bug can be reproduced consistently.
+      options:
+        - "Yes"
+        - "No"
+    validations:
+      required: true
+
+  - type: textarea
+    id: inputs
+    attributes:
+      label: Inputs Causing the Bug
+      description: Provide details about the inputs causing the issue.
+      placeholder: |
+        - URL(s): 
+        - Settings used: 
+        - Input data (if applicable): 
+      render: bash
+  
+  - type: textarea
+    id: steps_to_reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: Provide step-by-step instructions to reproduce the issue.
+      placeholder: |
+        1. Go to...
+        2. Click on...
+        3. Observe the issue...
+      render: bash
+  
+  - type: textarea
+    id: code_snippets
+    attributes:
+      label: Code snippets
+      description: Provide code snippets(if any). Add comments as necessary
+      placeholder: print("Hello world")
+      render: python
+
+  # Header Section with Title
+  - type: markdown
+    attributes:
+      value: |
+          ## Supporting Information
+          Please provide the following details to help us understand and resolve your issue. This will assist us in reproducing and diagnosing the problem
+
+  - type: input
+    id: os
+    attributes:
+      label: OS
+      description: Please provide the operating system & distro where the issue occurs.
+      placeholder: "e.g., Windows, macOS, Linux"
+    validations:
+      required: true
+
+  - type: input
+    id: python_version
+    attributes:
+      label: Python version
+      description: Specify the Python version being used.
+      placeholder: "e.g., 3.8.5"
+    validations:
+      required: true
+
+  # Browser Field
+  - type: input
+    id: browser
+    attributes:
+      label: Browser
+      description: Provide the name of the browser you are using.
+      placeholder: "e.g., Chrome, Firefox, Safari"
+    validations:
+      required: false
+
+  # Browser Version Field
+  - type: input
+    id: browser_version
+    attributes:
+      label: Browser version
+      description: Provide the version of the browser you are using.
+      placeholder: "e.g., 91.0.4472.124"
+    validations:
+      required: false
+
+  # Error Logs Field (Text Area)
+  - type: textarea
+    id: error_logs
+    attributes:
+      label: Error logs & Screenshots (if applicable)
+      description: If you encountered any errors, please provide the error logs. Attach any relevant screenshots to help us understand the issue.
+      placeholder: "Paste error logs here and attach your screenshots"
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Feature Requests
+    url: https://github.com/unclecode/crawl4ai/discussions/categories/feature-requests
+    about: "Suggest new features or enhancements for Crawl4AI"
+  - name: Forums - Q&A
+    url: https://github.com/unclecode/crawl4ai/discussions/categories/forums-q-a
+    about: "Ask questions or engage in general discussions about Crawl4AI"

.github/pull_request_template.md

@@ -0,0 +1,19 @@
+## Summary
+Please include a summary of the change and/or which issues are fixed.
+
+eg: `Fixes #123` (Tag GitHub issue numbers in this format, so it automatically links the issues with your PR)
+
+## List of files changed and why
+eg: quickstart.py - To update the example as per new changes
+
+## How Has This Been Tested?
+Please describe the tests that you ran to verify your changes.
+
+## Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] I have added/updated unit tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes

.gitignore

@@ -206,12 +206,52 @@ pypi_build.sh
 git_issues.py
 git_issues.md
 
+.next/
 .tests/
-.issues/
+# .issues/
 .docs/
 .issues/
 .gitboss/
 todo_executor.md
 protect-all-except-feature.sh
 manage-collab.sh
-publish.sh
+publish.sh
+combine.sh
+combined_output.txt
+.local
+.scripts
+tree.md
+tree.md
+.scripts
+.local
+.do
+/plans
+plans/
+
+# Codeium
+.codeiumignore
+todo/
+
+# Continue development files
+.continue/
+.continuerc.json
+continue.lock
+continue_core.log
+contextProviders/
+continue_workspace/
+.continue-cache/
+continue_config.json
+
+# Continue temporary files
+.continue-temp/
+.continue-logs/
+.continue-downloads/
+
+# Continue VS Code specific
+.vscode-continue/
+.vscode-continue-cache/
+
+.prompts/
+
+.llm.env
+.private/

CHANGELOG.md

@@ -1,5 +1,525 @@
 # Changelog
 
+All notable changes to Crawl4AI will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## Version 0.5.0 (2025-03-02)
+
+### Added
+
+- *(profiles)* Add BrowserProfiler class for dedicated browser profile management
+- *(cli)* Add interactive profile management to CLI with rich UI
+- *(profiles)* Add ability to crawl directly from profile management interface
+- *(browser)* Support identity-based browsing with persistent profiles
+- *(deep-crawling)* Add max_pages parameter to limit the number of pages crawled in all deep crawling strategies
+- *(deep-crawling)* Add score_threshold parameter to BFS and DFS strategies to filter URLs by score
+
+### Changed
+
+- *(browser)* Refactor profile management from ManagedBrowser to BrowserProfiler class
+- *(cli)* Enhance CLI with profile selection and status display for crawling
+- *(examples)* Update identity-based browsing example to use BrowserProfiler class
+- *(docs)* Update identity-based crawling documentation
+- *(docs)* Update deep crawling documentation with max_pages and score_threshold parameters
+- *(examples)* Add example demonstrating the use of max_pages and score_threshold parameters
+
+### Fixed
+
+- *(browser)* Fix profile detection and management on different platforms
+- *(cli)* Fix CLI command structure for better user experience
+- *(deep-crawling)* Improve BFS and DFS strategies to handle page count limits more efficiently
+
+
+## Version 0.5.0 (2025-02-21)
+
+### Added
+
+- *(crawler)* [**breaking**] Add memory-adaptive dispatcher with rate limiting
+- *(scraping)* [**breaking**] Add LXML-based scraping mode for improved performance
+- *(content-filter)* Add LLMContentFilter for intelligent markdown generation
+- *(dispatcher)* [**breaking**] Add streaming support for URL processing
+- *(browser)* [**breaking**] Improve browser context management and add shared data support
+- *(config)* [**breaking**] Add streaming support and config cloning
+- *(crawler)* Add URL redirection tracking
+- *(extraction)* Add LLM-powered schema generation utility
+- *(proxy)* Add proxy configuration support to CrawlerRunConfig
+- *(robots)* Add robots.txt compliance support
+- *(release)* [**breaking**] Prepare v0.4.3 beta release
+- *(proxy)* Add proxy rotation support and documentation
+- *(browser)* Add CDP URL configuration support
+- *(demo)* Uncomment feature demos and add fake-useragent dependency
+- *(pdf)* Add PDF processing capabilities
+- *(crawler)* [**breaking**] Enhance JavaScript execution and PDF processing
+- *(docker)* Add Docker deployment configuration and API server
+- *(docker)* Add Docker service integration and config serialization
+- *(docker)* [**breaking**] Enhance Docker deployment setup and configuration
+- *(api)* Improve cache handling and add API tests
+- *(crawler)* [**breaking**] Add deep crawling capabilities with BFS strategy
+- *(proxy)* [**breaking**] Add proxy rotation strategy
+- *(deep-crawling)* Add DFS strategy and update exports; refactor CLI entry point
+- *(cli)* Add command line interface with comprehensive features
+- *(config)* Enhance serialization and add deep crawling exports
+- *(crawler)* Add HTTP crawler strategy for lightweight web scraping
+- *(docker)* [**breaking**] Implement supervisor and secure API endpoints
+- *(docker)* [**breaking**] Add JWT authentication and improve server architecture
+
+### Changed
+
+- *(browser)* Update browser channel default to 'chromium' in BrowserConfig.from_args method
+- *(crawler)* Optimize response handling and default settings
+- *(crawler)* - Update hello_world example with proper content filtering
+- - Update hello_world.py example
+- *(docs)* [**breaking**] Reorganize documentation structure and update styles
+- *(dispatcher)* [**breaking**] Migrate to modular dispatcher system with enhanced monitoring
+- *(scraping)* [**breaking**] Replace ScrapingMode enum with strategy pattern
+- *(browser)* Improve browser path management
+- *(models)* Rename final_url to redirected_url for consistency
+- *(core)* [**breaking**] Improve type hints and remove unused file
+- *(docs)* Improve code formatting in features demo
+- *(user-agent)* Improve user agent generation system
+- *(core)* [**breaking**] Reorganize project structure and remove legacy code
+- *(docker)* Clean up import statements in server.py
+- *(docker)* Remove unused models and utilities for cleaner codebase
+- *(docker)* [**breaking**] Improve server architecture and configuration
+- *(deep-crawl)* [**breaking**] Reorganize deep crawling functionality into dedicated module
+- *(deep-crawling)* [**breaking**] Reorganize deep crawling strategies and add new implementations
+- *(crawling)* [**breaking**] Improve type hints and code cleanup
+- *(crawler)* [**breaking**] Improve HTML handling and cleanup codebase
+- *(crawler)* [**breaking**] Remove content filter functionality
+- *(examples)* Update API usage in features demo
+- *(config)* [**breaking**] Enhance serialization and config handling
+
+### Docs
+
+- Add Code of Conduct for the project (#410)
+
+### Documentation
+
+- *(extraction)* Add clarifying comments for CSS selector behavior
+- *(readme)* Update personal story and project vision
+- *(urls)* [**breaking**] Update documentation URLs to new domain
+- *(api)* Add streaming mode documentation and examples
+- *(readme)* Update version and feature announcements for v0.4.3b1
+- *(examples)* Update demo scripts and fix output formats
+- *(examples)* Update v0.4.3 features demo to v0.4.3b2
+- *(readme)* Update version references and fix links
+- *(multi-url)* [**breaking**] Improve documentation clarity and update examples
+- *(examples)* Update proxy rotation demo and disable other demos
+- *(api)* Improve formatting and readability of API documentation
+- *(examples)* Add SERP API project example
+- *(urls)* Update documentation URLs to new domain
+- *(readme)* Resolve merge conflict and update version info
+
+### Fixed
+
+- *(browser)* Update default browser channel to chromium and simplify channel selection logic
+- *(browser)* [**breaking**] Default to Chromium channel for new headless mode (#387)
+- *(browser)* Resolve merge conflicts in browser channel configuration
+- Prevent memory leaks by ensuring proper closure of Playwright pages
+- Not working long page screenshot (#403)
+- *(extraction)* JsonCss selector and crawler improvements
+- *(models)* [**breaking**] Make model fields optional with default values
+- *(dispatcher)* Adjust memory threshold and fix dispatcher initialization
+- *(install)* Ensure proper exit after running doctor command
+
+### Miscellaneous Tasks
+
+- *(cleanup)* Remove unused files and improve type hints
+- Add .gitattributes file
+
+## License Update
+
+Crawl4AI v0.5.0 updates the license to Apache 2.0 *with a required attribution clause*.  This means you are free to use, modify, and distribute Crawl4AI (even commercially), but you *must* clearly attribute the project in any public use or distribution.  See the updated `LICENSE` file for the full legal text and specific requirements.
+
+---
+
+## Version 0.4.3b2 (2025-01-21)
+
+This release introduces several powerful new features, including robots.txt compliance, dynamic proxy support, LLM-powered schema generation, and improved documentation.
+
+### Features
+
+-   **Robots.txt Compliance:**
+    -   Added robots.txt compliance support with efficient SQLite-based caching.
+    -   New `check_robots_txt` parameter in `CrawlerRunConfig` to enable robots.txt checking before crawling a URL.
+    -   Automated robots.txt checking is now integrated into `AsyncWebCrawler` with 403 status codes for blocked URLs.
+    
+-   **Proxy Configuration:**
+    -   Added proxy configuration support to `CrawlerRunConfig`, allowing dynamic proxy settings per crawl request.
+    -   Updated documentation with examples for using proxy configuration in crawl operations.
+
+-   **LLM-Powered Schema Generation:**
+    -   Introduced a new utility for automatic CSS and XPath schema generation using OpenAI or Ollama models.
+    -   Added comprehensive documentation and examples for schema generation.
+    -   New prompt templates optimized for HTML schema analysis.
+
+-   **URL Redirection Tracking:**
+    -   Added URL redirection tracking to capture the final URL after any redirects.
+    -   The final URL is now available in the `redirected_url` field of the `AsyncCrawlResponse` object.
+
+-   **Enhanced Streamlined Documentation:**
+    -   Refactored and improved the documentation structure for clarity and ease of use.
+    -   Added detailed explanations of new features and updated examples.
+
+-   **Improved Browser Context Management:**
+    -   Enhanced the management of browser contexts and added shared data support.
+    -   Introduced the `shared_data` parameter in `CrawlerRunConfig` to pass data between hooks.
+
+-   **Memory Dispatcher System:**
+    -   Migrated to a memory dispatcher system with enhanced monitoring capabilities.
+    -   Introduced `MemoryAdaptiveDispatcher` and `SemaphoreDispatcher` for improved resource management.
+    -   Added `RateLimiter` for rate limiting support.
+    -   New `CrawlerMonitor` for real-time monitoring of crawler operations.
+
+-   **Streaming Support:**
+    -   Added streaming support for processing crawled URLs as they are processed.
+    -   Enabled streaming mode with the `stream` parameter in `CrawlerRunConfig`.
+
+-   **Content Scraping Strategy:**
+    -   Introduced a new `LXMLWebScrapingStrategy` for faster content scraping.
+    -   Added support for selecting the scraping strategy via the `scraping_strategy` parameter in `CrawlerRunConfig`.
+
+### Bug Fixes
+
+-   **Browser Path Management:**
+    -   Improved browser path management for consistent behavior across different environments.
+
+-   **Memory Threshold:**
+    -   Adjusted the default memory threshold to improve resource utilization.
+
+-   **Pydantic Model Fields:**
+    -   Made several model fields optional with default values to improve flexibility.
+
+### Refactor
+
+-   **Documentation Structure:**
+    -   Reorganized documentation structure to improve navigation and readability.
+    -   Updated styles and added new sections for advanced features.
+
+-   **Scraping Mode:**
+    -   Replaced the `ScrapingMode` enum with a strategy pattern for more flexible content scraping.
+
+-   **Version Update:**
+    -   Updated the version to `0.4.248`.
+
+-   **Code Cleanup:**
+    -   Removed unused files and improved type hints.
+    -   Applied Ruff corrections for code quality.
+
+-   **Updated dependencies:**
+    -   Updated dependencies to their latest versions to ensure compatibility and security.
+
+-   **Ignored certain patterns and directories:**
+    -   Updated `.gitignore` and `.codeiumignore` to ignore additional patterns and directories, streamlining the development environment.
+
+-   **Simplified Personal Story in README:**
+    -   Streamlined the personal story and project vision in the `README.md` for clarity.
+
+-   **Removed Deprecated Files:**
+    -   Deleted several deprecated files and examples that are no longer relevant.
+
+---
+**Previous Releases:**
+
+### 0.4.24x (2024-12-31)
+-   **Enhanced SSL & Security**: New SSL certificate handling with custom paths and validation options for secure crawling.
+-   **Smart Content Filtering**: Advanced filtering system with regex support and efficient chunking strategies.
+-   **Improved JSON Extraction**: Support for complex JSONPath, JSON-CSS, and Microdata extraction.
+-   **New Field Types**: Added `computed`, `conditional`, `aggregate`, and `template` field types.
+-   **Performance Boost**: Optimized caching, parallel processing, and memory management.
+-   **Better Error Handling**: Enhanced debugging capabilities with detailed error tracking.
+-   **Security Features**: Improved input validation and safe expression evaluation.
+
+### 0.4.247 (2025-01-06)
+
+#### Added
+- **Windows Event Loop Configuration**: Introduced a utility function `configure_windows_event_loop` to resolve `NotImplementedError` for asyncio subprocesses on Windows. ([#utils.py](crawl4ai/utils.py), [#tutorials/async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md))
+- **`page_need_scroll` Method**: Added a method to determine if a page requires scrolling before taking actions in `AsyncPlaywrightCrawlerStrategy`. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
+
+#### Changed
+- **Version Bump**: Updated the version from `0.4.246` to `0.4.247`. ([#__version__.py](crawl4ai/__version__.py))
+- **Improved Scrolling Logic**: Enhanced scrolling methods in `AsyncPlaywrightCrawlerStrategy` by adding a `scroll_delay` parameter for better control. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
+- **Markdown Generation Example**: Updated the `hello_world.py` example to reflect the latest API changes and better illustrate features. ([#examples/hello_world.py](docs/examples/hello_world.py))
+- **Documentation Update**: 
+  - Added Windows-specific instructions for handling asyncio event loops. ([#async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md))
+
+#### Removed
+- **Legacy Markdown Generation Code**: Removed outdated and unused code for markdown generation in `content_scraping_strategy.py`. ([#content_scraping_strategy.py](crawl4ai/content_scraping_strategy.py))
+
+#### Fixed
+- **Page Closing to Prevent Memory Leaks**:
+  - **Description**: Added a `finally` block to ensure pages are closed when no `session_id` is provided.
+  - **Impact**: Prevents memory leaks caused by lingering pages after a crawl.
+  - **File**: [`async_crawler_strategy.py`](crawl4ai/async_crawler_strategy.py)
+  - **Code**:
+    ```python
+    finally:
+        # If no session_id is given we should close the page
+        if not config.session_id:
+            await page.close()
+    ```
+- **Multiple Element Selection**: Modified `_get_elements` in `JsonCssExtractionStrategy` to return all matching elements instead of just the first one, ensuring comprehensive extraction. ([#extraction_strategy.py](crawl4ai/extraction_strategy.py))
+- **Error Handling in Scrolling**: Added robust error handling to ensure scrolling proceeds safely even if a configuration is missing. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
+
+## [0.4.267] - 2025 - 01 - 06
+
+### Added
+- **Windows Event Loop Configuration**: Introduced a utility function `configure_windows_event_loop` to resolve `NotImplementedError` for asyncio subprocesses on Windows. ([#utils.py](crawl4ai/utils.py), [#tutorials/async-webcrawler-basics.md](docs/md_v3/tutorials/async-webcrawler-basics.md))
+- **`page_need_scroll` Method**: Added a method to determine if a page requires scrolling before taking actions in `AsyncPlaywrightCrawlerStrategy`. ([#async_crawler_strategy.py](crawl4ai/async_crawler_strategy.py))
+
+## [0.4.24] - 2024-12-31
+
+### Added
+- **Browser and SSL Handling**
+  - SSL certificate validation options in extraction strategies
+  - Custom certificate paths support
+  - Configurable certificate validation skipping
+  - Enhanced response status code handling with retry logic
+
+- **Content Processing**
+  - New content filtering system with regex support
+  - Advanced chunking strategies for large content
+  - Memory-efficient parallel processing
+  - Configurable chunk size optimization
+
+- **JSON Extraction**
+  - Complex JSONPath expression support
+  - JSON-CSS and Microdata extraction
+  - RDFa parsing capabilities
+  - Advanced data transformation pipeline
+
+- **Field Types**
+  - New field types: `computed`, `conditional`, `aggregate`, `template`
+  - Field inheritance system
+  - Reusable field definitions
+  - Custom validation rules
+
+### Changed
+- **Performance**
+  - Optimized selector compilation with caching
+  - Improved HTML parsing efficiency
+  - Enhanced memory management for large documents
+  - Batch processing optimizations
+
+- **Error Handling**
+  - More detailed error messages and categorization
+  - Enhanced debugging capabilities
+  - Improved performance metrics tracking
+  - Better error recovery mechanisms
+
+### Deprecated
+- Old field computation method using `eval`
+- Direct browser manipulation without proper SSL handling
+- Simple text-based content filtering
+
+### Removed
+- Legacy extraction patterns without proper error handling
+- Unsafe eval-based field computation
+- Direct DOM manipulation without sanitization
+
+### Fixed
+- Memory leaks in large document processing
+- SSL certificate validation issues
+- Incorrect handling of nested JSON structures
+- Performance bottlenecks in parallel processing
+
+### Security
+- Improved input validation and sanitization
+- Safe expression evaluation system
+- Enhanced resource protection
+- Rate limiting implementation
+
+## [0.4.1] - 2024-12-08
+
+### **File: `crawl4ai/async_crawler_strategy.py`**
+
+#### **New Parameters and Attributes Added**
+- **`text_mode` (boolean)**: Enables text-only mode, disables images, JavaScript, and GPU-related features for faster, minimal rendering.
+- **`light_mode` (boolean)**: Optimizes the browser by disabling unnecessary background processes and features for efficiency.
+- **`viewport_width` and `viewport_height`**: Dynamically adjusts based on `text_mode` mode (default values: 800x600 for `text_mode`, 1920x1080 otherwise).
+- **`extra_args`**: Adds browser-specific flags for `text_mode` mode.
+- **`adjust_viewport_to_content`**: Dynamically adjusts the viewport to the content size for accurate rendering.
+
+#### **Browser Context Adjustments**
+- Added **`viewport` adjustments**: Dynamically computed based on `text_mode` or custom configuration.
+- Enhanced support for `light_mode` and `text_mode` by adding specific browser arguments to reduce resource consumption.
+
+#### **Dynamic Content Handling**
+- **Full Page Scan Feature**:
+  - Scrolls through the entire page while dynamically detecting content changes.
+  - Ensures scrolling stops when no new dynamic content is loaded.
+
+#### **Session Management**
+- Added **`create_session`** method:
+  - Creates a new browser session and assigns a unique ID.
+  - Supports persistent and non-persistent contexts with full compatibility for cookies, headers, and proxies.
+
+#### **Improved Content Loading and Adjustment**
+- **`adjust_viewport_to_content`**:
+  - Automatically adjusts viewport to match content dimensions.
+  - Includes scaling via Chrome DevTools Protocol (CDP).
+- Enhanced content loading:
+  - Waits for images to load and ensures network activity is idle before proceeding.
+
+#### **Error Handling and Logging**
+- Improved error handling and detailed logging for:
+  - Viewport adjustment (`adjust_viewport_to_content`).
+  - Full page scanning (`scan_full_page`).
+  - Dynamic content loading.
+
+#### **Refactoring and Cleanup**
+- Removed hardcoded viewport dimensions in multiple places, replaced with dynamic values (`self.viewport_width`, `self.viewport_height`).
+- Removed commented-out and unused code for better readability.
+- Added default value for `delay_before_return_html` parameter.
+
+#### **Optimizations**
+- Reduced resource usage in `light_mode` by disabling unnecessary browser features such as extensions, background timers, and sync.
+- Improved compatibility for different browser types (`chrome`, `firefox`, `webkit`).
+
+---
+
+### **File: `docs/examples/quickstart_async.py`**
+
+#### **Schema Adjustment**
+- Changed schema reference for `LLMExtractionStrategy`:
+  - **Old**: `OpenAIModelFee.schema()`
+  - **New**: `OpenAIModelFee.model_json_schema()`
+  - This likely ensures better compatibility with the `OpenAIModelFee` class and its JSON schema.
+
+#### **Documentation Comments Updated**
+- Improved extraction instruction for schema-based LLM strategies.
+
+---
+
+### **New Features Added**
+1. **Text-Only Mode**:
+   - Focuses on minimal resource usage by disabling non-essential browser features.
+2. **Light Mode**:
+   - Optimizes browser for performance by disabling background tasks and unnecessary services.
+3. **Full Page Scanning**:
+   - Ensures the entire content of a page is crawled, including dynamic elements loaded during scrolling.
+4. **Dynamic Viewport Adjustment**:
+   - Automatically resizes the viewport to match content dimensions, improving compatibility and rendering accuracy.
+5. **Session Management**:
+   - Simplifies session handling with better support for persistent and non-persistent contexts.
+
+---
+
+### **Bug Fixes**
+- Fixed potential viewport mismatches by ensuring consistent use of `self.viewport_width` and `self.viewport_height` throughout the code.
+- Improved robustness of dynamic content loading to avoid timeouts and failed evaluations.
+
+## [0.3.75] December 1, 2024
+
+### PruningContentFilter
+
+#### 1. Introduced PruningContentFilter (Dec 01, 2024) (Dec 01, 2024)
+A new content filtering strategy that removes less relevant nodes based on metrics like text and link density.
+
+**Affected Files:**
+- `crawl4ai/content_filter_strategy.py`: Enhancement of content filtering capabilities.
+```diff
+Implemented effective pruning algorithm with comprehensive scoring.
+```
+- `README.md`: Improved documentation regarding new features.
+```diff
+Updated to include usage and explanation for the PruningContentFilter.
+```
+- `docs/md_v2/basic/content_filtering.md`: Expanded documentation for users.
+```diff
+Added detailed section explaining the PruningContentFilter.
+```
+
+#### 2. Added Unit Tests for PruningContentFilter (Dec 01, 2024) (Dec 01, 2024)
+Comprehensive tests added to ensure correct functionality of PruningContentFilter
+
+**Affected Files:**
+- `tests/async/test_content_filter_prune.py`: Increased test coverage for content filtering strategies.
+```diff
+Created test cases for various scenarios using the PruningContentFilter.
+```
+
+### Development Updates
+
+#### 3. Enhanced BM25ContentFilter tests (Dec 01, 2024) (Dec 01, 2024)
+Extended testing to cover additional edge cases and performance metrics.
+
+**Affected Files:**
+- `tests/async/test_content_filter_bm25.py`: Improved reliability and performance assurance.
+```diff
+Added tests for new extraction scenarios including malformed HTML.
+```
+
+### Infrastructure & Documentation
+
+#### 4. Updated Examples (Dec 01, 2024) (Dec 01, 2024)
+Altered examples in documentation to promote the use of PruningContentFilter alongside existing strategies.
+
+**Affected Files:**
+- `docs/examples/quickstart_async.py`: Enhanced usability and clarity for new users.
+- Revised example to illustrate usage of PruningContentFilter.
+
+## [0.3.746] November 29, 2024
+
+### Major Features
+1. Enhanced Docker Support (Nov 29, 2024)
+   - Improved GPU support in Docker images.
+   - Dockerfile refactored for better platform-specific installations.
+   - Introduced new Docker commands for different platforms:
+     - `basic-amd64`, `all-amd64`, `gpu-amd64` for AMD64.
+     - `basic-arm64`, `all-arm64`, `gpu-arm64` for ARM64.
+
+### Infrastructure & Documentation
+- Enhanced README.md to improve user guidance and installation instructions.
+- Added installation instructions for Playwright setup in README.
+- Created and updated examples in `docs/examples/quickstart_async.py` to be more useful and user-friendly.
+- Updated `requirements.txt` with a new `pydantic` dependency.
+- Bumped version number in `crawl4ai/__version__.py` to 0.3.746.
+
+### Breaking Changes
+- Streamlined application structure:
+  - Removed static pages and related code from `main.py` which might affect existing deployments relying on static content.
+
+### Development Updates
+- Developed `post_install` method in `crawl4ai/install.py` to streamline post-installation setup tasks.
+- Refined migration processes in `crawl4ai/migrations.py` with enhanced logging for better error visibility.
+- Updated `docker-compose.yml` to support local and hub services for different architectures, enhancing build and deploy capabilities.
+- Refactored example test cases in `docs/examples/docker_example.py` to facilitate comprehensive testing.
+
+### README.md
+Updated README with new docker commands and setup instructions.
+Enhanced installation instructions and guidance.
+
+### crawl4ai/install.py
+Added post-install script functionality.
+Introduced `post_install` method for automation of post-installation tasks.
+
+### crawl4ai/migrations.py
+Improved migration logging.
+Refined migration processes and added better logging.
+
+### docker-compose.yml
+Refactored docker-compose for better service management.
+Updated to define services for different platforms and versions.
+
+### requirements.txt
+Updated dependencies.
+Added `pydantic` to requirements file.
+
+### crawler/__version__.py
+Updated version number.
+Bumped version number to 0.3.746.
+
+### docs/examples/quickstart_async.py
+Enhanced example scripts.
+Uncommented example usage in async guide for user functionality.
+
+### main.py
+Refactored code to improve maintainability.
+Streamlined app structure by removing static pages code.
+
 ## [0.3.743] November 27, 2024
 
 Enhance features and documentation
@@ -514,7 +1034,7 @@ This commit introduces several key enhancements, including improved error handli
 - Improved `AsyncPlaywrightCrawlerStrategy.close()` method to use a shorter sleep time (0.5 seconds instead of 500), significantly reducing wait time when closing the crawler.
 - Enhanced flexibility in `CosineStrategy`:
   - Now uses a more generic `load_HF_embedding_model` function, allowing for easier swapping of embedding models.
-- Updated `JsonCssExtractionStrategy` and `JsonXPATHExtractionStrategy` for better JSON-based extraction.
+- Updated `JsonCssExtractionStrategy` and `JsonXPathExtractionStrategy` for better JSON-based extraction.
 
 ### Fixed
 - Addressed potential issues with the sliding window chunking strategy to ensure all text is properly chunked.
@@ -785,6 +1305,6 @@ These changes focus on refining the existing codebase, resulting in a more stabl
 - Maintaining the semantic context of inline tags (e.g., abbreviation, DEL, INS) for improved LLM-friendliness.
 - Updated Dockerfile to ensure compatibility across multiple platforms (Hopefully!).
 
-## [0.2.4] - 2024-06-17
+## [v0.2.4] - 2024-06-17
 ### Fixed
 - Fix issue #22: Use MD5 hash for caching HTML files to handle long URLs

CODE_OF_CONDUCT.md

@@ -0,0 +1,131 @@
+# Crawl4AI Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+unclecode@crawl4ai.com. All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTORS.md

@@ -6,7 +6,7 @@ We would like to thank the following people for their contributions to Crawl4AI:
 
 - [Unclecode](https://github.com/unclecode) - Project Creator and Main Developer
 - [Nasrin](https://github.com/ntohidi) - Project Manager and Developer
-- [Aravind Karnam](https://github.com/aravindkarnam) - Developer
+- [Aravind Karnam](https://github.com/aravindkarnam) - Head of Community and Product 
 
 ## Community Contributors
 
@@ -18,9 +18,19 @@ We would like to thank the following people for their contributions to Crawl4AI:
 
 ## Pull Requests
 
+- [dvschuyl](https://github.com/dvschuyl) - AsyncPlaywrightCrawlerStrategy page-evaluate context destroyed by navigation [#304](https://github.com/unclecode/crawl4ai/pull/304)
 - [nelzomal](https://github.com/nelzomal) - Enhance development installation instructions [#286](https://github.com/unclecode/crawl4ai/pull/286)
 - [HamzaFarhan](https://github.com/HamzaFarhan) - Handled the cases where markdown_with_citations, references_markdown, and filtered_html might not be defined [#293](https://github.com/unclecode/crawl4ai/pull/293)
 - [NanmiCoder](https://github.com/NanmiCoder) - fix: crawler strategy exception handling and fixes [#271](https://github.com/unclecode/crawl4ai/pull/271)
+- [paulokuong](https://github.com/paulokuong) - fix: RAWL4_AI_BASE_DIRECTORY should be Path object instead of string [#298](https://github.com/unclecode/crawl4ai/pull/298)
+
+#### Feb-Alpha-1
+- [sufianuddin](https://github.com/sufianuddin) - fix: [Documentation for JsonCssExtractionStrategy](https://github.com/unclecode/crawl4ai/issues/651)
+- [tautikAg](https://github.com/tautikAg) - fix: [Markdown output has incorect spacing](https://github.com/unclecode/crawl4ai/issues/599)
+- [cardit1](https://github.com/cardit1) - fix: ['AsyncPlaywrightCrawlerStrategy' object has no attribute 'downloads_path'](https://github.com/unclecode/crawl4ai/issues/585)
+- [dmurat](https://github.com/dmurat) - fix: [ Incorrect rendering of inline code inside of links ](https://github.com/unclecode/crawl4ai/issues/583)
+- [Sparshsing](https://github.com/Sparshsing) - fix: [Relative Urls in the webpage not extracted properly ](https://github.com/unclecode/crawl4ai/issues/570)
+
 
 
 ## Other Contributors
@@ -29,6 +39,11 @@ We would like to thank the following people for their contributions to Crawl4AI:
 - [Shiv Kumar](https://github.com/shivkumar0757)
 - [QIN2DIM](https://github.com/QIN2DIM)
 
+#### Typo fixes
+- [ssoydan](https://github.com/ssoydan)
+- [Darshan](https://github.com/Darshan2104)
+- [tuhinmallick](https://github.com/tuhinmallick)
+
 ## Acknowledgements
 
 We also want to thank all the users who have reported bugs, suggested features, or helped in any other way to make Crawl4AI better.

Dockerfile

@@ -1,29 +1,31 @@
-# syntax=docker/dockerfile:1.4
+FROM python:3.10-slim
 
-# Build arguments
-ARG PYTHON_VERSION=3.10
+# Set build arguments
+ARG APP_HOME=/app
+ARG GITHUB_REPO=https://github.com/unclecode/crawl4ai.git
+ARG GITHUB_BRANCH=main
+ARG USE_LOCAL=true
 
-# Base stage with system dependencies
-FROM python:${PYTHON_VERSION}-slim as base
-
-# Declare ARG variables again within the build stage
-ARG INSTALL_TYPE=all
-ARG ENABLE_GPU=false
-
-# Platform-specific labels
-LABEL maintainer="unclecode"
-LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
-LABEL version="1.0"
-
-# Environment setup
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
+ENV PYTHONFAULTHANDLER=1 \
+    PYTHONHASHSEED=random \
+    PYTHONUNBUFFERED=1 \
     PIP_NO_CACHE_DIR=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_DEFAULT_TIMEOUT=100 \
-    DEBIAN_FRONTEND=noninteractive
+    DEBIAN_FRONTEND=noninteractive \
+    REDIS_HOST=localhost \
+    REDIS_PORT=6379
+
+ARG PYTHON_VERSION=3.10
+ARG INSTALL_TYPE=default
+ARG ENABLE_GPU=false
+ARG TARGETARCH
+
+LABEL maintainer="unclecode"
+LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
+LABEL version="1.0"    
 
-# Install system dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     curl \
@@ -34,10 +36,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     pkg-config \
     python3-dev \
     libjpeg-dev \
-    libpng-dev \
+    redis-server \
+    supervisor \
     && rm -rf /var/lib/apt/lists/*
 
-# Playwright system dependencies for Linux
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libglib2.0-0 \
     libnss3 \
@@ -62,28 +64,50 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libatspi2.0-0 \
     && rm -rf /var/lib/apt/lists/*
 
-# GPU support if enabled and architecture is supported
-RUN if [ "$ENABLE_GPU" = "true" ] && [ "$(dpkg --print-architecture)" != "arm64" ] ; then \
-        apt-get update && apt-get install -y --no-install-recommends \
-        nvidia-cuda-toolkit \
-        && rm -rf /var/lib/apt/lists/* ; \
-    else \
-        echo "Skipping NVIDIA CUDA Toolkit installation (unsupported architecture or GPU disabled)"; \
-    fi
+RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \
+    apt-get update && apt-get install -y --no-install-recommends \
+    nvidia-cuda-toolkit \
+    && rm -rf /var/lib/apt/lists/* ; \
+else \
+    echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
+fi
 
-# Create and set working directory
-WORKDIR /app
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
+    echo "🦾 Installing ARM-specific optimizations"; \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libopenblas-dev \
+    && rm -rf /var/lib/apt/lists/*; \
+elif [ "$TARGETARCH" = "amd64" ]; then \
+    echo "🖥️ Installing AMD64-specific optimizations"; \
+    apt-get update && apt-get install -y --no-install-recommends \
+    libomp-dev \
+    && rm -rf /var/lib/apt/lists/*; \
+else \
+    echo "Skipping platform-specific optimizations (unsupported platform)"; \
+fi
 
-# Copy the entire project
-COPY . .
+WORKDIR ${APP_HOME}
 
-# Install base requirements
+RUN echo '#!/bin/bash\n\
+if [ "$USE_LOCAL" = "true" ]; then\n\
+    echo "📦 Installing from local source..."\n\
+    pip install --no-cache-dir /tmp/project/\n\
+else\n\
+    echo "🌐 Installing from GitHub..."\n\
+    for i in {1..3}; do \n\
+        git clone --branch ${GITHUB_BRANCH} ${GITHUB_REPO} /tmp/crawl4ai && break || \n\
+        { echo "Attempt $i/3 failed! Taking a short break... ☕"; sleep 5; }; \n\
+    done\n\
+    pip install --no-cache-dir /tmp/crawl4ai\n\
+fi' > /tmp/install.sh && chmod +x /tmp/install.sh
+
+COPY . /tmp/project/
+
+COPY deploy/docker/supervisord.conf .
+
+COPY deploy/docker/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Install required library for FastAPI
-RUN pip install fastapi uvicorn psutil
-
-# Install ML dependencies first for better layer caching
 RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
         pip install --no-cache-dir \
             torch \
@@ -96,34 +120,37 @@ RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
         python -m nltk.downloader punkt stopwords ; \
     fi
 
-# Install the package
 RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
-        pip install ".[all]" && \
+        pip install "/tmp/project/[all]" && \
         python -m crawl4ai.model_loader ; \
     elif [ "$INSTALL_TYPE" = "torch" ] ; then \
-        pip install ".[torch]" ; \
+        pip install "/tmp/project/[torch]" ; \
     elif [ "$INSTALL_TYPE" = "transformer" ] ; then \
-        pip install ".[transformer]" && \
+        pip install "/tmp/project/[transformer]" && \
         python -m crawl4ai.model_loader ; \
     else \
-        pip install "." ; \
+        pip install "/tmp/project" ; \
     fi
+    
+RUN pip install --no-cache-dir --upgrade pip && \
+    /tmp/install.sh && \
+    python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
+    python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')"
+    
+RUN playwright install --with-deps chromium
 
-    # Install MkDocs and required plugins
-RUN pip install --no-cache-dir \
-    mkdocs \
-    mkdocs-material \
-    mkdocs-terminal \
-    pymdown-extensions
+COPY deploy/docker/* ${APP_HOME}/
 
-# Build MkDocs documentation
-RUN mkdocs build
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD bash -c '\
+    MEM=$(free -m | awk "/^Mem:/{print \$2}"); \
+    if [ $MEM -lt 2048 ]; then \
+        echo "⚠️ Warning: Less than 2GB RAM available! Your container might need a memory boost! 🚀"; \
+        exit 1; \
+    fi && \
+    redis-cli ping > /dev/null && \
+    curl -f http://localhost:8000/health || exit 1'
 
-# Install Playwright and browsers
-RUN playwright install
-
-# Expose port
-EXPOSE 8000 11235 9222 8080
-
-# Start the FastAPI server
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "11235"]
+EXPOSE 6379
+CMD ["supervisord", "-c", "supervisord.conf"]
+    

LICENSE

@@ -48,4 +48,22 @@ You may add Your own copyright statement to Your modifications and may provide a
 
 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
 
-END OF TERMS AND CONDITIONS
+END OF TERMS AND CONDITIONS
+
+---
+Attribution Requirement
+
+All distributions, publications, or public uses of this software, or derivative works based on this software, must include the following attribution:
+
+"This product includes software developed by UncleCode (https://x.com/unclecode) as part of the Crawl4AI project (https://github.com/unclecode/crawl4ai)."
+
+This attribution must be displayed in a prominent and easily accessible location, such as:
+
+-   For software distributions: In a NOTICE file, README file, or equivalent documentation.
+-   For publications (research papers, articles, blog posts): In the acknowledgments section or a footnote.
+-   For websites/web applications: In an "About" or "Credits" section.
+-   For command-line tools: In the help/usage output.
+
+This requirement ensures proper credit is given for the use of Crawl4AI and helps promote the project.
+
+---

MANIFEST.in

@@ -1 +1,2 @@
-include requirements.txt
+include requirements.txt
+recursive-include crawl4ai/js_snippet *.js

README.md

@@ -1,17 +1,41 @@
-# 🔥🕷️ Crawl4AI: Crawl Smarter, Faster, Freely. For AI.
+# 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper.
+
+<div align="center">
 
 <a href="https://trendshift.io/repositories/11716" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11716" alt="unclecode%2Fcrawl4ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 
 [![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
-![PyPI - Downloads](https://img.shields.io/pypi/dm/Crawl4AI)
 [![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members)
-[![GitHub Issues](https://img.shields.io/github/issues/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/issues)
-[![GitHub Pull Requests](https://img.shields.io/github/issues-pr/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/pulls)
+
+[![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai)
+[![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai)](https://pypi.org/project/crawl4ai/)
+[![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
+
+<!-- [![Documentation Status](https://readthedocs.org/projects/crawl4ai/badge/?version=latest)](https://crawl4ai.readthedocs.io/) -->
 [![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md)
+
+</div>
 
 Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for LLMs, AI agents, and data pipelines. Open source, flexible, and built for real-time performance, Crawl4AI empowers developers with unmatched speed, precision, and deployment ease.  
 
-[✨ Check out what's new in the latest update!](#recent-updates)  
+[✨ Check out latest update v0.5.0](#-recent-updates)
+
+🎉 **Version 0.5.0 is out!** This major release introduces Deep Crawling with BFS/DFS/BestFirst strategies, Memory-Adaptive Dispatcher, Multiple Crawling Strategies (Playwright and HTTP), Docker Deployment with FastAPI, Command-Line Interface (CLI), and more! [Read the release notes →](https://docs.crawl4ai.com/blog)
+
+<details>
+<summary>🤓 <strong>My Personal Story</strong></summary>
+
+My journey with computers started in childhood when my dad, a computer scientist, introduced me to an Amstrad computer. Those early days sparked a fascination with technology, leading me to pursue computer science and specialize in NLP during my postgraduate studies. It was during this time that I first delved into web crawling, building tools to help researchers organize papers and extract information from publications a challenging yet rewarding experience that honed my skills in data extraction.
+
+Fast forward to 2023, I was working on a tool for a project and needed a crawler to convert a webpage into markdown. While exploring solutions, I found one that claimed to be open-source but required creating an account and generating an API token. Worse, it turned out to be a SaaS model charging $16, and its quality didn’t meet my standards. Frustrated, I realized this was a deeper problem. That frustration turned into turbo anger mode, and I decided to build my own solution. In just a few days, I created Crawl4AI. To my surprise, it went viral, earning thousands of GitHub stars and resonating with a global community.
+
+I made Crawl4AI open-source for two reasons. First, it’s my way of giving back to the open-source community that has supported me throughout my career. Second, I believe data should be accessible to everyone, not locked behind paywalls or monopolized by a few. Open access to data lays the foundation for the democratization of AI, a vision where individuals can train their own models and take ownership of their information. This library is the first step in a larger journey to create the best open-source data extraction and generation tool the world has ever seen, built collaboratively by a passionate community.
+
+Thank you to everyone who has supported this project, used it, and shared feedback. Your encouragement motivates me to dream even bigger. Join us, file issues, submit PRs, or spread the word. Together, we can build a tool that truly empowers people to access their own data and reshape the future of AI.
+</details>
 
 ## 🧐 Why Crawl4AI?
 
@@ -26,24 +50,52 @@ Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant
 
 1. Install Crawl4AI:
 ```bash
-pip install crawl4ai
+# Install the package
+pip install -U crawl4ai
+
+# For pre release versions
+pip install crawl4ai --pre
+
+# Run post-installation setup
+crawl4ai-setup
+
+# Verify your installation
+crawl4ai-doctor
 ```
 
-2. Run a simple web crawl:
+If you encounter any browser-related issues, you can install them manually:
+```bash
+python -m playwright install --with-deps chromium
+```
+
+2. Run a simple web crawl with Python:
 ```python
 import asyncio
-from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai import *
 
 async def main():
-    async with AsyncWebCrawler(verbose=True) as crawler:
-        result = await crawler.arun(url="https://www.nbcnews.com/business")
-        # Soone will be change to result.markdown
-        print(result.markdown_v2.raw_markdown) 
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url="https://www.nbcnews.com/business",
+        )
+        print(result.markdown)
 
 if __name__ == "__main__":
     asyncio.run(main())
 ```
 
+3. Or use the new command-line interface:
+```bash
+# Basic crawl with markdown output
+crwl https://www.nbcnews.com/business -o markdown
+
+# Deep crawl with BFS strategy, max 10 pages
+crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10
+
+# Use LLM extraction with a specific question
+crwl https://www.example.com/products -q "Extract all product prices"
+```
+
 ## ✨ Features 
 
 <details>
@@ -72,10 +124,12 @@ if __name__ == "__main__":
 
 - 🖥️ **Managed Browser**: Use user-owned browsers with full control, avoiding bot detection.
 - 🔄 **Remote Browser Control**: Connect to Chrome Developer Tools Protocol for remote, large-scale data extraction.
+- 👤 **Browser Profiler**: Create and manage persistent profiles with saved authentication states, cookies, and settings.
 - 🔒 **Session Management**: Preserve browser states and reuse them for multi-step crawling.
 - 🧩 **Proxy Support**: Seamlessly connect to proxies with authentication for secure access.
 - ⚙️ **Full Browser Control**: Modify headers, cookies, user agents, and more for tailored crawling setups.
 - 🌍 **Multi-Browser Support**: Compatible with Chromium, Firefox, and WebKit.
+- 📐 **Dynamic Viewport Adjustment**: Automatically adjusts the browser viewport to match page content, ensuring complete rendering and capturing of all elements.
 
 </details>
 
@@ -91,16 +145,19 @@ if __name__ == "__main__":
 - 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches.
 - 📄 **Metadata Extraction**: Retrieve structured metadata from web pages.
 - 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content.
+- 🕵️ **Lazy Load Handling**: Waits for images to fully load, ensuring no content is missed due to lazy loading.
+- 🔄 **Full-Page Scanning**: Simulates scrolling to load and capture all dynamic content, perfect for infinite scroll pages.
 
 </details>
 
 <details>
 <summary>🚀 <strong>Deployment</strong></summary>
 
-- 🐳 **Dockerized Setup**: Optimized Docker image with API server for easy deployment.
+- 🐳 **Dockerized Setup**: Optimized Docker image with FastAPI server for easy deployment.
+- 🔑 **Secure Authentication**: Built-in JWT token authentication for API security.
 - 🔄 **API Gateway**: One-click deployment with secure token authentication for API-based workflows.
 - 🌐 **Scalable Architecture**: Designed for mass-scale production and optimized server performance.
-- ⚙️ **DigitalOcean Deployment**: Ready-to-deploy configurations for DigitalOcean and similar platforms.
+- ☁️ **Cloud Deployment**: Ready-to-deploy configurations for major cloud platforms.
 
 </details>
 
@@ -117,41 +174,11 @@ if __name__ == "__main__":
 
 </details>
 
-
-
 ## Try it Now!
 
 ✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing)
 
-✨ Visit our [Documentation Website](https://crawl4ai.com/mkdocs/)
-
-## Features ✨
-
-- 🆓 Completely free and open-source
-- 🚀 Blazing fast performance, outperforming many paid services
-- 🤖 LLM-friendly output formats (JSON, cleaned HTML, markdown)
-- 🌐 Multi-browser support (Chromium, Firefox, WebKit)
-- 🌍 Supports crawling multiple URLs simultaneously
-- 🎨 Extracts and returns all media tags (Images, Audio, and Video)
-- 🔗 Extracts all external and internal links
-- 📚 Extracts metadata from the page
-- 🔄 Custom hooks for authentication, headers, and page modifications
-- 🕵️ User-agent customization
-- 🖼️ Takes screenshots of pages with enhanced error handling
-- 📜 Executes multiple custom JavaScripts before crawling
-- 📊 Generates structured output without LLM using JsonCssExtractionStrategy
-- 📚 Various chunking strategies: topic-based, regex, sentence, and more
-- 🧠 Advanced extraction strategies: cosine clustering, LLM, and more
-- 🎯 CSS selector support for precise data extraction
-- 📝 Passes instructions/keywords to refine extraction
-- 🔒 Proxy support with authentication for enhanced access
-- 🔄 Session management for complex multi-page crawling
-- 🌐 Asynchronous architecture for improved performance
-- 🖼️ Improved image processing with lazy-loading detection
-- 🕰️ Enhanced handling of delayed content loading
-- 🔑 Custom headers support for LLM interactions
-- 🖼️ iframe content extraction for comprehensive analysis
-- ⏱️ Flexible timeout and delayed content retrieval options
+✨ Visit our [Documentation Website](https://docs.crawl4ai.com/)
 
 ## Installation 🛠️
 
@@ -168,11 +195,12 @@ For basic web crawling and scraping tasks:
 
 ```bash
 pip install crawl4ai
+crawl4ai-setup # Setup the browser
 ```
 
 By default, this will install the asynchronous version of Crawl4AI, using Playwright for web crawling.
 
-👉 **Note**: When you install Crawl4AI, the setup script should automatically install and set up Playwright. However, if you encounter any Playwright-related errors, you can manually install it using one of these methods:
+👉 **Note**: When you install Crawl4AI, the `crawl4ai-setup` should automatically install and set up Playwright. However, if you encounter any Playwright-related errors, you can manually install it using one of these methods:
 
 1. Through the command line:
 
@@ -223,71 +251,29 @@ pip install -e ".[all]"             # Install all optional features
 </details>
 
 <details>
-<summary>🚀 <strong>One-Click Deployment</strong></summary>
+<summary>🐳 <strong>Docker Deployment</strong></summary>
 
-Deploy your own instance of Crawl4AI with one click:
+> 🚀 **Major Changes Coming!** We're developing a completely new Docker implementation that will make deployment even more efficient and seamless. The current Docker setup is being deprecated in favor of this new solution.
 
-[![DigitalOcean Referral Badge](https://web-platforms.sfo2.cdn.digitaloceanspaces.com/WWW/Badge%203.svg)](https://www.digitalocean.com/?repo=https://github.com/unclecode/crawl4ai/tree/0.3.74&refcode=a0780f1bdb3d&utm_campaign=Referral_Invite&utm_medium=Referral_Program&utm_source=badge)
+### Current Docker Support
 
-> 💡 **Recommended specs**: 4GB RAM minimum. Select "professional-xs" or higher when deploying for stable operation.
+The existing Docker implementation is being deprecated and will be replaced soon. If you still need to use Docker with the current version:
 
-The deploy will:
-- Set up a Docker container with Crawl4AI
-- Configure Playwright and all dependencies
-- Start the FastAPI server on port `11235`
-- Set up health checks and auto-deployment
+- 📚 [Deprecated Docker Setup](./docs/deprecated/docker-deployment.md) - Instructions for the current Docker implementation
+- ⚠️ Note: This setup will be replaced in the next major release
+
+### What's Coming Next?
+
+Our new Docker implementation will bring:
+- Improved performance and resource efficiency
+- Streamlined deployment process
+- Better integration with Crawl4AI features
+- Enhanced scalability options
+
+Stay connected with our [GitHub repository](https://github.com/unclecode/crawl4ai) for updates!
 
 </details>
 
-<details>
-<summary>🐳 <strong>Using Docker</strong></summary>
-
-Crawl4AI is available as Docker images for easy deployment. You can either pull directly from Docker Hub (recommended) or build from the repository.
-
----
-
-### Option 1: Docker Hub (Recommended)
-
-```bash
-# Pull and run from Docker Hub (choose one):
-docker pull unclecode/crawl4ai:basic    # Basic crawling features
-docker pull unclecode/crawl4ai:all      # Full installation (ML, LLM support)
-docker pull unclecode/crawl4ai:gpu      # GPU-enabled version
-
-# Run the container
-docker run -p 11235:11235 unclecode/crawl4ai:basic  # Replace 'basic' with your chosen version
-
-# In case you want to set platform to arm64
-docker run --platform linux/arm64 -p 11235:11235 unclecode/crawl4ai:basic
-
-# In case to allocate more shared memory for the container
-docker run --shm-size=2gb -p 11235:11235 unclecode/crawl4ai:basic
-```
-
----
-
-### Option 2: Build from Repository
-
-```bash
-# Clone the repository
-git clone https://github.com/unclecode/crawl4ai.git
-cd crawl4ai
-
-# Build the image
-docker build -t crawl4ai:local \
-  --build-arg INSTALL_TYPE=basic \  # Options: basic, all
-  .
-
-# In case you want to set platform to arm64
-docker build -t crawl4ai:local \
-  --build-arg INSTALL_TYPE=basic \  # Options: basic, all
-  --platform linux/arm64 \
-  .
-
-# Run your local build
-docker run -p 11235:11235 crawl4ai:local
-```
-
 ---
 
 ### Quick Test
@@ -304,11 +290,11 @@ response = requests.post(
 )
 task_id = response.json()["task_id"]
 
-# Get results
+# Continue polling until the task is complete (status="completed")
 result = requests.get(f"http://localhost:11235/task/{task_id}")
 ```
 
-For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://crawl4ai.com/mkdocs/basic/docker-deployment/).
+For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://docs.crawl4ai.com/basic/docker-deployment/).
 
 </details>
 
@@ -322,25 +308,32 @@ You can check the project structure in the directory [https://github.com/uncleco
 
 ```python
 import asyncio
-from crawl4ai import AsyncWebCrawler, CacheMode
-from crawl4ai.content_filter_strategy import BM25ContentFilter
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
 from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
 
 async def main():
-    async with AsyncWebCrawler(
+    browser_config = BrowserConfig(
         headless=True,  
         verbose=True,
-    ) as crawler:
+    )
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.ENABLED,
+        markdown_generator=DefaultMarkdownGenerator(
+            content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed", min_word_threshold=0)
+        ),
+        # markdown_generator=DefaultMarkdownGenerator(
+        #     content_filter=BM25ContentFilter(user_query="WHEN_WE_FOCUS_BASED_ON_A_USER_QUERY", bm25_threshold=1.0)
+        # ),
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
         result = await crawler.arun(
             url="https://docs.micronaut.io/4.7.6/guide/",
-            cache_mode=CacheMode.ENABLED,
-            markdown_generator=DefaultMarkdownGenerator(
-                content_filter=BM25ContentFilter(user_query=None, bm25_threshold=1.0)
-            ),
+            config=run_config
         )
-        print(len(result.markdown))
-        print(len(result.fit_markdown))
-        print(len(result.markdown_v2.fit_markdown))
+        print(len(result.markdown.raw_markdown))
+        print(len(result.markdown.fit_markdown))
 
 if __name__ == "__main__":
     asyncio.run(main())
@@ -353,7 +346,7 @@ if __name__ == "__main__":
 
 ```python
 import asyncio
-from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
 import json
 
@@ -388,36 +381,26 @@ async def main():
             "type": "attribute",
             "attribute": "src"
         }
-    ]
+    }
 }
 
     extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
 
-    async with AsyncWebCrawler(
+    browser_config = BrowserConfig(
         headless=False,
         verbose=True
-    ) as crawler:
+    )
+    run_config = CrawlerRunConfig(
+        extraction_strategy=extraction_strategy,
+        js_code=["""(async () => {const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");for(let tab of tabs) {tab.scrollIntoView();tab.click();await new Promise(r => setTimeout(r, 500));}})();"""],
+        cache_mode=CacheMode.BYPASS
+    )
+        
+    async with AsyncWebCrawler(config=browser_config) as crawler:
         
-        # Create the JavaScript that handles clicking multiple times
-        js_click_tabs = """
-        (async () => {
-            const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");
-            
-            for(let tab of tabs) {
-                // scroll to the tab
-                tab.scrollIntoView();
-                tab.click();
-                // Wait for content to load and animations to complete
-                await new Promise(r => setTimeout(r, 500));
-            }
-        })();
-        """     
-
         result = await crawler.arun(
             url="https://www.kidocode.com/degrees/technology",
-            extraction_strategy=JsonCssExtractionStrategy(schema, verbose=True),
-            js_code=[js_click_tabs],
-            cache_mode=CacheMode.BYPASS
+            config=run_config
         )
 
         companies = json.loads(result.extracted_content)
@@ -437,7 +420,7 @@ if __name__ == "__main__":
 ```python
 import os
 import asyncio
-from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LlmConfig
 from crawl4ai.extraction_strategy import LLMExtractionStrategy
 from pydantic import BaseModel, Field
 
@@ -447,21 +430,26 @@ class OpenAIModelFee(BaseModel):
     output_fee: str = Field(..., description="Fee for output token for the OpenAI model.")
 
 async def main():
-    async with AsyncWebCrawler(verbose=True) as crawler:
+    browser_config = BrowserConfig(verbose=True)
+    run_config = CrawlerRunConfig(
+        word_count_threshold=1,
+        extraction_strategy=LLMExtractionStrategy(
+            # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2
+            # provider="ollama/qwen2", api_token="no-token", 
+            llmConfig = LlmConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')), 
+            schema=OpenAIModelFee.schema(),
+            extraction_type="schema",
+            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+            Do not miss any models in the entire content. One extracted model JSON format should look like this: 
+            {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""
+        ),            
+        cache_mode=CacheMode.BYPASS,
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
         result = await crawler.arun(
             url='https://openai.com/api/pricing/',
-            word_count_threshold=1,
-            extraction_strategy=LLMExtractionStrategy(
-                # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2
-                # provider="ollama/qwen2", api_token="no-token", 
-                provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY'), 
-                schema=OpenAIModelFee.schema(),
-                extraction_type="schema",
-                instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
-                Do not miss any models in the entire content. One extracted model JSON format should look like this: 
-                {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""
-            ),            
-            cache_mode=CacheMode.BYPASS,
+            config=run_config
         )
         print(result.extracted_content)
 
@@ -472,43 +460,35 @@ if __name__ == "__main__":
 </details>
 
 <details>
-<summary>🤖 <strong>Using You own Browswer with Custome User Profile</strong></summary>
+<summary>🤖 <strong>Using You own Browser with Custom User Profile</strong></summary>
 
 ```python
 import os, sys
 from pathlib import Path
 import asyncio, time
-from crawl4ai import AsyncWebCrawler
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
 
 async def test_news_crawl():
     # Create a persistent user data directory
     user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
     os.makedirs(user_data_dir, exist_ok=True)
 
-    async with AsyncWebCrawler(
+    browser_config = BrowserConfig(
         verbose=True,
         headless=True,
         user_data_dir=user_data_dir,
         use_persistent_context=True,
-        headers={
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.5",
-            "Accept-Encoding": "gzip, deflate, br",
-            "DNT": "1",
-            "Connection": "keep-alive",
-            "Upgrade-Insecure-Requests": "1",
-            "Sec-Fetch-Dest": "document",
-            "Sec-Fetch-Mode": "navigate",
-            "Sec-Fetch-Site": "none",
-            "Sec-Fetch-User": "?1",
-            "Cache-Control": "max-age=0",
-        }
-    ) as crawler:
+    )
+    run_config = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS
+    )
+    
+    async with AsyncWebCrawler(config=browser_config) as crawler:
         url = "ADDRESS_OF_A_CHALLENGING_WEBSITE"
         
         result = await crawler.arun(
             url,
-            cache_mode=CacheMode.BYPASS,
+            config=run_config,
             magic=True,
         )
         
@@ -518,21 +498,82 @@ async def test_news_crawl():
 
 </details>
 
+## ✨ Recent Updates
 
-## ✨ Recent Updates   
+### Version 0.5.0 Major Release Highlights
 
-- 🚀 **Improved ManagedBrowser Configuration**: Dynamic host and port support for more flexible browser management.  
-- 📝 **Enhanced Markdown Generation**: New generator class for better formatting and customization.  
-- ⚡ **Fast HTML Formatting**: Significantly optimized HTML formatting in the web crawler.  
-- 🛠️ **Utility & Sanitization Upgrades**: Improved sanitization and expanded utility functions for streamlined workflows.  
-- 👥 **Acknowledgments**: Added contributor details and pull request acknowledgments for better transparency.  
+-   **🚀 Deep Crawling System**: Explore websites beyond initial URLs with three strategies:
+    -   **BFS Strategy**: Breadth-first search explores websites level by level
+    -   **DFS Strategy**: Depth-first search explores each branch deeply before backtracking
+    -   **BestFirst Strategy**: Uses scoring functions to prioritize which URLs to crawl next
+    -   **Page Limiting**: Control the maximum number of pages to crawl with `max_pages` parameter
+    -   **Score Thresholds**: Filter URLs based on relevance scores
+-   **⚡ Memory-Adaptive Dispatcher**: Dynamically adjusts concurrency based on system memory with built-in rate limiting
+-   **🔄 Multiple Crawling Strategies**:
+    -   **AsyncPlaywrightCrawlerStrategy**: Browser-based crawling with JavaScript support (Default)
+    -   **AsyncHTTPCrawlerStrategy**: Fast, lightweight HTTP-only crawler for simple tasks
+-   **🐳 Docker Deployment**: Easy deployment with FastAPI server and streaming/non-streaming endpoints
+-   **💻 Command-Line Interface**: New `crwl` CLI provides convenient terminal access to all features with intuitive commands and configuration options
+-   **👤 Browser Profiler**: Create and manage persistent browser profiles to save authentication states, cookies, and settings for seamless crawling of protected content
+-   **🧠 Crawl4AI Coding Assistant**: AI-powered coding assistant to answer your question for Crawl4ai, and generate proper code for crawling.
+-   **🏎️ LXML Scraping Mode**: Fast HTML parsing using the `lxml` library for improved performance
+-   **🌐 Proxy Rotation**: Built-in support for proxy switching with `RoundRobinProxyStrategy`
+-   **🤖 LLM Content Filter**: Intelligent markdown generation using LLMs
+-   **📄 PDF Processing**: Extract text, images, and metadata from PDF files
+-   **🔗 URL Redirection Tracking**: Automatically follow and record HTTP redirects
+-   **🤖 LLM Schema Generation**: Easily create extraction schemas with LLM assistance
+-   **🔍 robots.txt Compliance**: Respect website crawling rules
 
+Read the full details in our [0.5.0 Release Notes](https://docs.crawl4ai.com/blog/releases/0.5.0.html) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
+
+## Version Numbering in Crawl4AI
+
+Crawl4AI follows standard Python version numbering conventions (PEP 440) to help users understand the stability and features of each release.
+
+### Version Numbers Explained
+
+Our version numbers follow this pattern: `MAJOR.MINOR.PATCH` (e.g., 0.4.3)
+
+#### Pre-release Versions
+We use different suffixes to indicate development stages:
+
+- `dev` (0.4.3dev1): Development versions, unstable
+- `a` (0.4.3a1): Alpha releases, experimental features
+- `b` (0.4.3b1): Beta releases, feature complete but needs testing
+- `rc` (0.4.3rc1): Release candidates, potential final version
+
+#### Installation
+- Regular installation (stable version):
+  ```bash
+  pip install -U crawl4ai
+  ```
+
+- Install pre-release versions:
+  ```bash
+  pip install crawl4ai --pre
+  ```
+
+- Install specific version:
+  ```bash
+  pip install crawl4ai==0.4.3b1
+  ```
+
+#### Why Pre-releases?
+We use pre-releases to:
+- Test new features in real-world scenarios
+- Gather feedback before final releases
+- Ensure stability for production users
+- Allow early adopters to try new features
+
+For production environments, we recommend using the stable version. For testing new features, you can opt-in to pre-releases using the `--pre` flag.
 
 ## 📖 Documentation & Roadmap 
 
-For detailed documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://crawl4ai.com/mkdocs/).
+> 🚨 **Documentation Update Alert**: We're undertaking a major documentation overhaul next week to reflect recent updates and improvements. Stay tuned for a more comprehensive and up-to-date guide!
 
-Moreover to check our development plans and upcoming features, check out our [Roadmap](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md).
+For current documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://docs.crawl4ai.com/).
+
+To check our development plans and upcoming features, visit our [Roadmap](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md).
 
 <details>
 <summary>📈 <strong>Development TODOs</strong></summary>
@@ -554,11 +595,85 @@ Moreover to check our development plans and upcoming features, check out our [Ro
 
 ## 🤝 Contributing 
 
-We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTING.md) for more information.
+We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information.
 
-## 📄 License 
+I'll help modify the license section with badges. For the halftone effect, here's a version with it:
 
-Crawl4AI is released under the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE).
+Here's the updated license section:
+
+## 📄 License & Attribution
+
+This project is licensed under the Apache License 2.0 with a required attribution clause. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
+
+### Attribution Requirements
+When using Crawl4AI, you must include one of the following attribution methods:
+
+#### 1. Badge Attribution (Recommended)
+Add one of these badges to your README, documentation, or website:
+
+| Theme | Badge |
+|-------|-------|
+| **Disco Theme (Animated)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Night Theme (Dark with Neon)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Dark Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+| **Light Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/></a> |
+ 
+
+HTML code for adding the badges:
+```html
+<!-- Disco Theme (Animated) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Night Theme (Dark with Neon) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Dark Theme (Classic) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Light Theme (Classic) -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/>
+</a>
+
+<!-- Simple Shield Badge -->
+<a href="https://github.com/unclecode/crawl4ai">
+  <img src="https://img.shields.io/badge/Powered%20by-Crawl4AI-blue?style=flat-square" alt="Powered by Crawl4AI"/>
+</a>
+```
+
+#### 2. Text Attribution
+Add this line to your documentation:
+```
+This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
+```
+
+## 📚 Citation
+
+If you use Crawl4AI in your research or project, please cite:
+
+```bibtex
+@software{crawl4ai2024,
+  author = {UncleCode},
+  title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper},
+  year = {2024},
+  publisher = {GitHub},
+  journal = {GitHub Repository},
+  howpublished = {\url{https://github.com/unclecode/crawl4ai}},
+  commit = {Please use the commit hash you're working with}
+}
+```
+
+Text citation format:
+```
+UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software]. 
+GitHub. https://github.com/unclecode/crawl4ai
+```
 
 ## 📧 Contact 
 
@@ -595,9 +710,6 @@ We envision a future where AI is powered by real human knowledge, ensuring data
 For more details, see our [full mission statement](./MISSION.md).
 </details>
 
-
-
-
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

README.sync.md

@@ -1,244 +0,0 @@
-# Crawl4AI v0.2.77 🕷️🤖
-
-[![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
-[![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members)
-[![GitHub Issues](https://img.shields.io/github/issues/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/issues)
-[![GitHub Pull Requests](https://img.shields.io/github/issues-pr/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/pulls)
-[![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)
-
-Crawl4AI simplifies web crawling and data extraction, making it accessible for large language models (LLMs) and AI applications. 🆓🌐
-
-#### [v0.2.77] - 2024-08-02
-
-Major improvements in functionality, performance, and cross-platform compatibility! 🚀
-
-- 🐳 **Docker enhancements**:
-  - Significantly improved Dockerfile for easy installation on Linux, Mac, and Windows.
-- 🌐 **Official Docker Hub image**:
-  - Launched our first official image on Docker Hub for streamlined deployment (unclecode/crawl4ai).
-- 🔧 **Selenium upgrade**:
-  - Removed dependency on ChromeDriver, now using Selenium's built-in capabilities for better compatibility.
-- 🖼️ **Image description**:
-  - Implemented ability to generate textual descriptions for extracted images from web pages.
-- ⚡ **Performance boost**:
-  - Various improvements to enhance overall speed and performance.
-  
-## Try it Now!
-
-✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1sJPAmeLj5PMrg2VgOwMJ2ubGIcK0cJeX?usp=sharing)
-
-✨ visit our [Documentation Website](https://crawl4ai.com/mkdocs/)
-
-✨ Check [Demo](https://crawl4ai.com/mkdocs/demo)
-
-## Features ✨
-
-- 🆓 Completely free and open-source
-- 🤖 LLM-friendly output formats (JSON, cleaned HTML, markdown)
-- 🌍 Supports crawling multiple URLs simultaneously
-- 🎨 Extracts and returns all media tags (Images, Audio, and Video)
-- 🔗 Extracts all external and internal links
-- 📚 Extracts metadata from the page
-- 🔄 Custom hooks for authentication, headers, and page modifications before crawling
-- 🕵️ User-agent customization
-- 🖼️ Takes screenshots of the page
-- 📜 Executes multiple custom JavaScripts before crawling
-- 📚 Various chunking strategies: topic-based, regex, sentence, and more
-- 🧠 Advanced extraction strategies: cosine clustering, LLM, and more
-- 🎯 CSS selector support
-- 📝 Passes instructions/keywords to refine extraction
-
-# Crawl4AI
-
-## 🌟 Shoutout to Contributors of v0.2.77!
-
-A big thank you to the amazing contributors who've made this release possible:
-
-- [@aravindkarnam](https://github.com/aravindkarnam) for the new image description feature
-- [@FractalMind](https://github.com/FractalMind) for our official Docker Hub image
-- [@ketonkss4](https://github.com/ketonkss4) for helping streamline our Selenium setup
-
-Your contributions are driving Crawl4AI forward! 🚀
-
-## Cool Examples 🚀
-
-### Quick Start
-
-```python
-from crawl4ai import WebCrawler
-
-# Create an instance of WebCrawler
-crawler = WebCrawler()
-
-# Warm up the crawler (load necessary models)
-crawler.warmup()
-
-# Run the crawler on a URL
-result = crawler.run(url="https://www.nbcnews.com/business")
-
-# Print the extracted content
-print(result.markdown)
-```
-
-## How to install 🛠 
-
-### Using pip 🐍
-```bash
-virtualenv venv
-source venv/bin/activate
-pip install "crawl4ai @ git+https://github.com/unclecode/crawl4ai.git"
-```
-
-### Using Docker 🐳
-
-```bash
-# For Mac users (M1/M2)
-# docker build --platform linux/amd64 -t crawl4ai .
-docker build -t crawl4ai .
-docker run -d -p 8000:80 crawl4ai
-```
-
-### Using Docker Hub 🐳
-
-```bash
-docker pull unclecode/crawl4ai:latest
-docker run -d -p 8000:80 unclecode/crawl4ai:latest
-```
-
-
-## Speed-First Design 🚀
-
-Perhaps the most important design principle for this library is speed. We need to ensure it can handle many links and resources in parallel as quickly as possible. By combining this speed with fast LLMs like Groq, the results will be truly amazing.
-
-```python
-import time
-from crawl4ai.web_crawler import WebCrawler
-crawler = WebCrawler()
-crawler.warmup()
-
-start = time.time()
-url = r"https://www.nbcnews.com/business"
-result = crawler.run( url, word_count_threshold=10, bypass_cache=True)
-end = time.time()
-print(f"Time taken: {end - start}")
-```
-
-Let's take a look the calculated time for the above code snippet:
-
-```bash
-[LOG] 🚀 Crawling done, success: True, time taken: 1.3623387813568115 seconds
-[LOG] 🚀 Content extracted, success: True, time taken: 0.05715131759643555 seconds
-[LOG] 🚀 Extraction, time taken: 0.05750393867492676 seconds.
-Time taken: 1.439958095550537
-```
-Fetching the content from the page took 1.3623 seconds, and extracting the content took 0.0575 seconds. 🚀
-
-### Extract Structured Data from Web Pages 📊
-
-Crawl all OpenAI models and their fees from the official page.
-
-```python
-import os
-from crawl4ai import WebCrawler
-from crawl4ai.extraction_strategy import LLMExtractionStrategy
-from pydantic import BaseModel, Field
-
-class OpenAIModelFee(BaseModel):
-    model_name: str = Field(..., description="Name of the OpenAI model.")
-    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
-    output_fee: str = Field(..., description="Fee for output token ßfor the OpenAI model.")
-
-url = 'https://openai.com/api/pricing/'
-crawler = WebCrawler()
-crawler.warmup()
-
-result = crawler.run(
-        url=url,
-        word_count_threshold=1,
-        extraction_strategy= LLMExtractionStrategy(
-            provider= "openai/gpt-4o", api_token = os.getenv('OPENAI_API_KEY'), 
-            schema=OpenAIModelFee.schema(),
-            extraction_type="schema",
-            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
-            Do not miss any models in the entire content. One extracted model JSON format should look like this: 
-            {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""
-        ),            
-        bypass_cache=True,
-    )
-
-print(result.extracted_content)
-```
-
-### Execute JS, Filter Data with CSS Selector, and Clustering
-
-```python
-from crawl4ai import WebCrawler
-from crawl4ai.chunking_strategy import CosineStrategy
-
-js_code = ["const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"]
-
-crawler = WebCrawler()
-crawler.warmup()
-
-result = crawler.run(
-    url="https://www.nbcnews.com/business",
-    js=js_code,
-    css_selector="p",
-    extraction_strategy=CosineStrategy(semantic_filter="technology")
-)
-
-print(result.extracted_content)
-```
-
-### Extract Structured Data from Web Pages With Proxy and BaseUrl
-
-```python
-from crawl4ai import WebCrawler
-from crawl4ai.extraction_strategy import LLMExtractionStrategy
-
-def create_crawler():
-    crawler = WebCrawler(verbose=True, proxy="http://127.0.0.1:7890")
-    crawler.warmup()
-    return crawler
-
-crawler = create_crawler()
-
-crawler.warmup()
-
-result = crawler.run(
-    url="https://www.nbcnews.com/business",
-    extraction_strategy=LLMExtractionStrategy(
-        provider="openai/gpt-4o",
-        api_token="sk-",
-        base_url="https://api.openai.com/v1"
-    )
-)
-
-print(result.markdown)
-```
-
-## Documentation 📚
-
-For detailed documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://crawl4ai.com/mkdocs/).
-
-## Contributing 🤝
-
-We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTING.md) for more information.
-
-## License 📄
-
-Crawl4AI is released under the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE).
-
-## Contact 📧
-
-For questions, suggestions, or feedback, feel free to reach out:
-
-- GitHub: [unclecode](https://github.com/unclecode)
-- Twitter: [@unclecode](https://twitter.com/unclecode)
-- Website: [crawl4ai.com](https://crawl4ai.com)
-
-Happy Crawling! 🕸️🚀
-
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

cliff.toml

@@ -0,0 +1,24 @@
+[changelog]
+# Template format
+header = """
+# Changelog\n
+All notable changes to this project will be documented in this file.\n
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n
+"""
+
+# Organize commits by type
+[git]
+conventional_commits = true
+filter_unconventional = true
+commit_parsers = [
+    { message = "^feat", group = "Added"},
+    { message = "^fix", group = "Fixed"},
+    { message = "^doc", group = "Documentation"},
+    { message = "^perf", group = "Performance"},
+    { message = "^refactor", group = "Changed"},
+    { message = "^style", group = "Changed"},
+    { message = "^test", group = "Testing"},
+    { message = "^chore\\(release\\): prepare for", skip = true},
+    { message = "^chore", group = "Miscellaneous Tasks"},
+]

crawl4ai/__init__.py

@@ -1,32 +1,147 @@
 # __init__.py
+import warnings
 
 from .async_webcrawler import AsyncWebCrawler, CacheMode
-
-from .models import CrawlResult
-from .__version__ import __version__
-# __version__ = "0.3.73"
+from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig
+from .content_scraping_strategy import (
+    ContentScrapingStrategy,
+    WebScrapingStrategy,
+    LXMLWebScrapingStrategy,
+)
+from .async_logger import (
+    AsyncLoggerBase,
+    AsyncLogger,
+)
+from .proxy_strategy import (
+    ProxyRotationStrategy,
+    RoundRobinProxyStrategy,
+)
+from .extraction_strategy import (
+    ExtractionStrategy,
+    LLMExtractionStrategy,
+    CosineStrategy,
+    JsonCssExtractionStrategy,
+    JsonXPathExtractionStrategy,
+)
+from .chunking_strategy import ChunkingStrategy, RegexChunking
+from .markdown_generation_strategy import DefaultMarkdownGenerator
+from .content_filter_strategy import (
+    PruningContentFilter,
+    BM25ContentFilter,
+    LLMContentFilter,
+    RelevantContentFilter,
+)
+from .models import CrawlResult, MarkdownGenerationResult
+from .async_dispatcher import (
+    MemoryAdaptiveDispatcher,
+    SemaphoreDispatcher,
+    RateLimiter,
+    CrawlerMonitor,
+    DisplayMode,
+    BaseDispatcher,
+)
+from .docker_client import Crawl4aiDockerClient
+from .hub import CrawlerHub
+from .browser_profiler import BrowserProfiler
+from .deep_crawling import (
+    DeepCrawlStrategy,
+    BFSDeepCrawlStrategy,
+    FilterChain,
+    ContentTypeFilter,
+    DomainFilter,
+    URLFilter,
+    FilterStats,
+    SEOFilter,
+    KeywordRelevanceScorer,
+    URLScorer,
+    CompositeScorer,
+    DomainAuthorityScorer,
+    FreshnessScorer,
+    PathDepthScorer,
+    BestFirstCrawlingStrategy,
+    DFSDeepCrawlStrategy,
+    DeepCrawlDecorator,
+)
 
 __all__ = [
+    "AsyncLoggerBase",
+    "AsyncLogger",
     "AsyncWebCrawler",
+    "BrowserProfiler",
+    "DeepCrawlStrategy",
+    "BFSDeepCrawlStrategy",
+    "BestFirstCrawlingStrategy",
+    "DFSDeepCrawlStrategy",
+    "FilterChain",
+    "ContentTypeFilter",
+    "DomainFilter",
+    "FilterStats",
+    "URLFilter",
+    "SEOFilter",
+    "KeywordRelevanceScorer",
+    "URLScorer",
+    "CompositeScorer",
+    "DomainAuthorityScorer",
+    "FreshnessScorer",
+    "PathDepthScorer",
+    "DeepCrawlDecorator",
     "CrawlResult",
+    "CrawlerHub",
     "CacheMode",
+    "ContentScrapingStrategy",
+    "WebScrapingStrategy",
+    "LXMLWebScrapingStrategy",
+    "BrowserConfig",
+    "CrawlerRunConfig",
+    "HTTPCrawlerConfig",
+    "ExtractionStrategy",
+    "LLMExtractionStrategy",
+    "CosineStrategy",
+    "JsonCssExtractionStrategy",
+    "JsonXPathExtractionStrategy",
+    "ChunkingStrategy",
+    "RegexChunking",
+    "DefaultMarkdownGenerator",
+    "RelevantContentFilter",
+    "PruningContentFilter",
+    "BM25ContentFilter",
+    "LLMContentFilter",
+    "BaseDispatcher",
+    "MemoryAdaptiveDispatcher",
+    "SemaphoreDispatcher",
+    "RateLimiter",
+    "CrawlerMonitor",
+    "DisplayMode",
+    "MarkdownGenerationResult",
+    "Crawl4aiDockerClient",
+    "ProxyRotationStrategy",
+    "RoundRobinProxyStrategy",
 ]
 
-def is_sync_version_installed():
-    try:
-        import selenium
-        return True
-    except ImportError:
-        return False
 
-if is_sync_version_installed():
-    try:
-        from .web_crawler import WebCrawler
-        __all__.append("WebCrawler")
-    except ImportError:
-        import warnings
-        print("Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies.")
-else:
-    WebCrawler = None
-    # import warnings
-    # print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.")
+# def is_sync_version_installed():
+#     try:
+#         import selenium # noqa
+
+#         return True
+#     except ImportError:
+#         return False
+
+
+# if is_sync_version_installed():
+#     try:
+#         from .web_crawler import WebCrawler
+
+#         __all__.append("WebCrawler")
+#     except ImportError:
+#         print(
+#             "Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies."
+#         )
+# else:
+#     WebCrawler = None
+#     # import warnings
+#     # print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.")
+
+# Disable all Pydantic warnings
+warnings.filterwarnings("ignore", module="pydantic")
+# pydantic_warnings.filter_warnings()

crawl4ai/__version__.py

@@ -1,2 +1,2 @@
 # crawl4ai/_version.py
-__version__ = "0.3.744"
+__version__ = "0.5.0.post1"

crawl4ai/async_database.py

@@ -2,25 +2,29 @@ import os
 from pathlib import Path
 import aiosqlite
 import asyncio
-from typing import Optional, Tuple, Dict
+from typing import Optional, Dict
 from contextlib import asynccontextmanager
 import logging
 import json  # Added for serialization/deserialization
 from .utils import ensure_content_dirs, generate_content_hash
-from .models import CrawlResult
-import xxhash
+from .models import CrawlResult, MarkdownGenerationResult, StringCompatibleMarkdown
 import aiofiles
-from .config import NEED_MIGRATION
-from .version_manager import VersionManager
+from .utils import VersionManager
 from .async_logger import AsyncLogger
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
+from .utils import get_error_context, create_box_message
 
-base_directory = DB_PATH = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+# Set up logging
+# logging.basicConfig(level=logging.INFO)
+# logger = logging.getLogger(__name__)
+# logger.setLevel(logging.INFO)
+
+base_directory = DB_PATH = os.path.join(
+    os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
+)
 os.makedirs(DB_PATH, exist_ok=True)
 DB_PATH = os.path.join(base_directory, "crawl4ai.db")
 
+
 class AsyncDatabaseManager:
     def __init__(self, pool_size: int = 10, max_retries: int = 3):
         self.db_path = DB_PATH
@@ -31,28 +35,27 @@ class AsyncDatabaseManager:
         self.pool_lock = asyncio.Lock()
         self.init_lock = asyncio.Lock()
         self.connection_semaphore = asyncio.Semaphore(pool_size)
-        self._initialized = False  
+        self._initialized = False
         self.version_manager = VersionManager()
         self.logger = AsyncLogger(
             log_file=os.path.join(base_directory, ".crawl4ai", "crawler_db.log"),
             verbose=False,
-            tag_width=10
+            tag_width=10,
         )
-        
-        
+
     async def initialize(self):
         """Initialize the database and connection pool"""
         try:
             self.logger.info("Initializing database", tag="INIT")
             # Ensure the database file exists
             os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
-            
+
             # Check if version update is needed
             needs_update = self.version_manager.needs_update()
-            
+
             # Always ensure base table exists
             await self.ainit_db()
-            
+
             # Verify the table exists
             async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
                 async with db.execute(
@@ -61,33 +64,37 @@ class AsyncDatabaseManager:
                     result = await cursor.fetchone()
                     if not result:
                         raise Exception("crawled_data table was not created")
-            
+
             # If version changed or fresh install, run updates
             if needs_update:
                 self.logger.info("New version detected, running updates", tag="INIT")
                 await self.update_db_schema()
-                from .migrations import run_migration  # Import here to avoid circular imports
+                from .migrations import (
+                    run_migration,
+                )  # Import here to avoid circular imports
+
                 await run_migration()
                 self.version_manager.update_version()  # Update stored version after successful migration
-                self.logger.success("Version update completed successfully", tag="COMPLETE")
+                self.logger.success(
+                    "Version update completed successfully", tag="COMPLETE"
+                )
             else:
-                self.logger.success("Database initialization completed successfully", tag="COMPLETE")
+                self.logger.success(
+                    "Database initialization completed successfully", tag="COMPLETE"
+                )
 
-                
         except Exception as e:
             self.logger.error(
                 message="Database initialization error: {error}",
                 tag="ERROR",
-                params={"error": str(e)}
+                params={"error": str(e)},
             )
             self.logger.info(
-                message="Database will be initialized on first use",
-                tag="INIT"
+                message="Database will be initialized on first use", tag="INIT"
             )
-                        
+
             raise
 
-            
     async def cleanup(self):
         """Cleanup connections when shutting down"""
         async with self.pool_lock:
@@ -97,35 +104,97 @@ class AsyncDatabaseManager:
 
     @asynccontextmanager
     async def get_connection(self):
-        """Connection pool manager"""
+        """Connection pool manager with enhanced error handling"""
         if not self._initialized:
-            # Use an asyncio.Lock to ensure only one initialization occurs
             async with self.init_lock:
                 if not self._initialized:
-                    await self.initialize()
-                    self._initialized = True
+                    try:
+                        await self.initialize()
+                        self._initialized = True
+                    except Exception as e:
+                        import sys
+
+                        error_context = get_error_context(sys.exc_info())
+                        self.logger.error(
+                            message="Database initialization failed:\n{error}\n\nContext:\n{context}\n\nTraceback:\n{traceback}",
+                            tag="ERROR",
+                            force_verbose=True,
+                            params={
+                                "error": str(e),
+                                "context": error_context["code_context"],
+                                "traceback": error_context["full_traceback"],
+                            },
+                        )
+                        raise
 
         await self.connection_semaphore.acquire()
         task_id = id(asyncio.current_task())
+
         try:
             async with self.pool_lock:
                 if task_id not in self.connection_pool:
-                    conn = await aiosqlite.connect(
-                        self.db_path,
-                        timeout=30.0
-                    )
-                    await conn.execute('PRAGMA journal_mode = WAL')
-                    await conn.execute('PRAGMA busy_timeout = 5000')
-                    self.connection_pool[task_id] = conn
+                    try:
+                        conn = await aiosqlite.connect(self.db_path, timeout=30.0)
+                        await conn.execute("PRAGMA journal_mode = WAL")
+                        await conn.execute("PRAGMA busy_timeout = 5000")
+
+                        # Verify database structure
+                        async with conn.execute(
+                            "PRAGMA table_info(crawled_data)"
+                        ) as cursor:
+                            columns = await cursor.fetchall()
+                            column_names = [col[1] for col in columns]
+                            expected_columns = {
+                                "url",
+                                "html",
+                                "cleaned_html",
+                                "markdown",
+                                "extracted_content",
+                                "success",
+                                "media",
+                                "links",
+                                "metadata",
+                                "screenshot",
+                                "response_headers",
+                                "downloaded_files",
+                            }
+                            missing_columns = expected_columns - set(column_names)
+                            if missing_columns:
+                                raise ValueError(
+                                    f"Database missing columns: {missing_columns}"
+                                )
+
+                        self.connection_pool[task_id] = conn
+                    except Exception as e:
+                        import sys
+
+                        error_context = get_error_context(sys.exc_info())
+                        error_message = (
+                            f"Unexpected error in db get_connection at line {error_context['line_no']} "
+                            f"in {error_context['function']} ({error_context['filename']}):\n"
+                            f"Error: {str(e)}\n\n"
+                            f"Code context:\n{error_context['code_context']}"
+                        )
+                        self.logger.error(
+                            message=create_box_message(error_message, type="error"),
+                        )
+
+                        raise
 
             yield self.connection_pool[task_id]
 
         except Exception as e:
+            import sys
+
+            error_context = get_error_context(sys.exc_info())
+            error_message = (
+                f"Unexpected error in db get_connection at line {error_context['line_no']} "
+                f"in {error_context['function']} ({error_context['filename']}):\n"
+                f"Error: {str(e)}\n\n"
+                f"Code context:\n{error_context['code_context']}"
+            )
             self.logger.error(
-                message="Connection error: {error}",
-                tag="ERROR",
-                force_verbose=True,
-                params={"error": str(e)}
+                message=create_box_message(error_message, type="error"),
             )
             raise
         finally:
@@ -135,7 +204,6 @@ class AsyncDatabaseManager:
                     del self.connection_pool[task_id]
             self.connection_semaphore.release()
 
-
     async def execute_with_retry(self, operation, *args):
         """Execute database operations with retry logic"""
         for attempt in range(self.max_retries):
@@ -150,18 +218,16 @@ class AsyncDatabaseManager:
                         message="Operation failed after {retries} attempts: {error}",
                         tag="ERROR",
                         force_verbose=True,
-                        params={
-                            "retries": self.max_retries,
-                            "error": str(e)
-                        }
-                    )                    
+                        params={"retries": self.max_retries, "error": str(e)},
+                    )
                     raise
                 await asyncio.sleep(1 * (attempt + 1))  # Exponential backoff
 
     async def ainit_db(self):
         """Initialize database schema"""
         async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
-            await db.execute('''
+            await db.execute(
+                """
                 CREATE TABLE IF NOT EXISTS crawled_data (
                     url TEXT PRIMARY KEY,
                     html TEXT,
@@ -176,21 +242,27 @@ class AsyncDatabaseManager:
                     response_headers TEXT DEFAULT "{}",
                     downloaded_files TEXT DEFAULT "{}"  -- New column added
                 )
-            ''')
+            """
+            )
             await db.commit()
 
-        
-
     async def update_db_schema(self):
         """Update database schema if needed"""
         async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
             cursor = await db.execute("PRAGMA table_info(crawled_data)")
             columns = await cursor.fetchall()
             column_names = [column[1] for column in columns]
-            
+
             # List of new columns to add
-            new_columns = ['media', 'links', 'metadata', 'screenshot', 'response_headers', 'downloaded_files']
-            
+            new_columns = [
+                "media",
+                "links",
+                "metadata",
+                "screenshot",
+                "response_headers",
+                "downloaded_files",
+            ]
+
             for column in new_columns:
                 if column not in column_names:
                     await self.aalter_db_add_column(column, db)
@@ -198,69 +270,100 @@ class AsyncDatabaseManager:
 
     async def aalter_db_add_column(self, new_column: str, db):
         """Add new column to the database"""
-        if new_column == 'response_headers':
-            await db.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT "{{}}"')
+        if new_column == "response_headers":
+            await db.execute(
+                f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT "{{}}"'
+            )
         else:
-            await db.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""')
+            await db.execute(
+                f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""'
+            )
         self.logger.info(
             message="Added column '{column}' to the database",
             tag="INIT",
-            params={"column": new_column}
-        )        
-
+            params={"column": new_column},
+        )
 
     async def aget_cached_url(self, url: str) -> Optional[CrawlResult]:
         """Retrieve cached URL data as CrawlResult"""
+
         async def _get(db):
             async with db.execute(
-                'SELECT * FROM crawled_data WHERE url = ?', (url,)
+                "SELECT * FROM crawled_data WHERE url = ?", (url,)
             ) as cursor:
                 row = await cursor.fetchone()
                 if not row:
                     return None
-                    
+
                 # Get column names
                 columns = [description[0] for description in cursor.description]
                 # Create dict from row data
                 row_dict = dict(zip(columns, row))
-                
+
                 # Load content from files using stored hashes
                 content_fields = {
-                    'html': row_dict['html'],
-                    'cleaned_html': row_dict['cleaned_html'],
-                    'markdown': row_dict['markdown'],
-                    'extracted_content': row_dict['extracted_content'],
-                    'screenshot': row_dict['screenshot']
+                    "html": row_dict["html"],
+                    "cleaned_html": row_dict["cleaned_html"],
+                    "markdown": row_dict["markdown"],
+                    "extracted_content": row_dict["extracted_content"],
+                    "screenshot": row_dict["screenshot"],
+                    "screenshots": row_dict["screenshot"],
                 }
-                
+
                 for field, hash_value in content_fields.items():
                     if hash_value:
                         content = await self._load_content(
-                            hash_value, 
-                            field.split('_')[0]  # Get content type from field name
+                            hash_value,
+                            field.split("_")[0],  # Get content type from field name
                         )
                         row_dict[field] = content or ""
                     else:
                         row_dict[field] = ""
 
                 # Parse JSON fields
-                json_fields = ['media', 'links', 'metadata', 'response_headers']
+                json_fields = [
+                    "media",
+                    "links",
+                    "metadata",
+                    "response_headers",
+                    "markdown",
+                ]
                 for field in json_fields:
                     try:
-                        row_dict[field] = json.loads(row_dict[field]) if row_dict[field] else {}
+                        row_dict[field] = (
+                            json.loads(row_dict[field]) if row_dict[field] else {}
+                        )
                     except json.JSONDecodeError:
-                        row_dict[field] = {}
+                        # Very UGLY, never mention it to me please
+                        if field == "markdown" and isinstance(row_dict[field], str):
+                            row_dict[field] = MarkdownGenerationResult(
+                                raw_markdown=row_dict[field] or "",
+                                markdown_with_citations="",
+                                references_markdown="",
+                                fit_markdown="",
+                                fit_html="",
+                            )
+                        else:
+                            row_dict[field] = {}
+
+                if isinstance(row_dict["markdown"], Dict):
+                    if row_dict["markdown"].get("raw_markdown"):
+                        row_dict["markdown"] = row_dict["markdown"]["raw_markdown"]
 
                 # Parse downloaded_files
                 try:
-                    row_dict['downloaded_files'] = json.loads(row_dict['downloaded_files']) if row_dict['downloaded_files'] else []
+                    row_dict["downloaded_files"] = (
+                        json.loads(row_dict["downloaded_files"])
+                        if row_dict["downloaded_files"]
+                        else []
+                    )
                 except json.JSONDecodeError:
-                    row_dict['downloaded_files'] = []
+                    row_dict["downloaded_files"] = []
 
                 # Remove any fields not in CrawlResult model
                 valid_fields = CrawlResult.__annotations__.keys()
                 filtered_dict = {k: v for k, v in row_dict.items() if k in valid_fields}
-                
+                filtered_dict["markdown"] = row_dict["markdown"]
                 return CrawlResult(**filtered_dict)
 
         try:
@@ -270,7 +373,7 @@ class AsyncDatabaseManager:
                 message="Error retrieving cached URL: {error}",
                 tag="ERROR",
                 force_verbose=True,
-                params={"error": str(e)}
+                params={"error": str(e)},
             )
             return None
 
@@ -278,19 +381,52 @@ class AsyncDatabaseManager:
         """Cache CrawlResult data"""
         # Store content files and get hashes
         content_map = {
-            'html': (result.html, 'html'),
-            'cleaned_html': (result.cleaned_html or "", 'cleaned'),
-            'markdown': (result.markdown or "", 'markdown'),
-            'extracted_content': (result.extracted_content or "", 'extracted'),
-            'screenshot': (result.screenshot or "", 'screenshots')
+            "html": (result.html, "html"),
+            "cleaned_html": (result.cleaned_html or "", "cleaned"),
+            "markdown": None,
+            "extracted_content": (result.extracted_content or "", "extracted"),
+            "screenshot": (result.screenshot or "", "screenshots"),
         }
-        
+
+        try:
+            if isinstance(result.markdown, StringCompatibleMarkdown):
+                content_map["markdown"] = (
+                    result.markdown,
+                    "markdown",
+                )
+            elif isinstance(result.markdown, MarkdownGenerationResult):
+                content_map["markdown"] = (
+                    result.markdown.model_dump_json(),
+                    "markdown",
+                )
+            elif isinstance(result.markdown, str):
+                markdown_result = MarkdownGenerationResult(raw_markdown=result.markdown)
+                content_map["markdown"] = (
+                    markdown_result.model_dump_json(),
+                    "markdown",
+                )
+            else:
+                content_map["markdown"] = (
+                    MarkdownGenerationResult().model_dump_json(),
+                    "markdown",
+                )
+        except Exception as e:
+            self.logger.warning(
+                message=f"Error processing markdown content: {str(e)}", tag="WARNING"
+            )
+            # Fallback to empty markdown result
+            content_map["markdown"] = (
+                MarkdownGenerationResult().model_dump_json(),
+                "markdown",
+            )
+
         content_hashes = {}
         for field, (content, content_type) in content_map.items():
             content_hashes[field] = await self._store_content(content, content_type)
 
         async def _cache(db):
-            await db.execute('''
+            await db.execute(
+                """
                 INSERT INTO crawled_data (
                     url, html, cleaned_html, markdown,
                     extracted_content, success, media, links, metadata,
@@ -309,20 +445,22 @@ class AsyncDatabaseManager:
                     screenshot = excluded.screenshot,
                     response_headers = excluded.response_headers,
                     downloaded_files = excluded.downloaded_files
-            ''', (
-                result.url,
-                content_hashes['html'],
-                content_hashes['cleaned_html'],
-                content_hashes['markdown'],
-                content_hashes['extracted_content'],
-                result.success,
-                json.dumps(result.media),
-                json.dumps(result.links),
-                json.dumps(result.metadata or {}),
-                content_hashes['screenshot'],
-                json.dumps(result.response_headers or {}),
-                json.dumps(result.downloaded_files or [])
-            ))
+            """,
+                (
+                    result.url,
+                    content_hashes["html"],
+                    content_hashes["cleaned_html"],
+                    content_hashes["markdown"],
+                    content_hashes["extracted_content"],
+                    result.success,
+                    json.dumps(result.media),
+                    json.dumps(result.links),
+                    json.dumps(result.metadata or {}),
+                    content_hashes["screenshot"],
+                    json.dumps(result.response_headers or {}),
+                    json.dumps(result.downloaded_files or []),
+                ),
+            )
 
         try:
             await self.execute_with_retry(_cache)
@@ -331,14 +469,14 @@ class AsyncDatabaseManager:
                 message="Error caching URL: {error}",
                 tag="ERROR",
                 force_verbose=True,
-                params={"error": str(e)}
+                params={"error": str(e)},
             )
-            
 
     async def aget_total_count(self) -> int:
         """Get total number of cached URLs"""
+
         async def _count(db):
-            async with db.execute('SELECT COUNT(*) FROM crawled_data') as cursor:
+            async with db.execute("SELECT COUNT(*) FROM crawled_data") as cursor:
                 result = await cursor.fetchone()
                 return result[0] if result else 0
 
@@ -349,14 +487,15 @@ class AsyncDatabaseManager:
                 message="Error getting total count: {error}",
                 tag="ERROR",
                 force_verbose=True,
-                params={"error": str(e)}
+                params={"error": str(e)},
             )
             return 0
 
     async def aclear_db(self):
         """Clear all data from the database"""
+
         async def _clear(db):
-            await db.execute('DELETE FROM crawled_data')
+            await db.execute("DELETE FROM crawled_data")
 
         try:
             await self.execute_with_retry(_clear)
@@ -365,13 +504,14 @@ class AsyncDatabaseManager:
                 message="Error clearing database: {error}",
                 tag="ERROR",
                 force_verbose=True,
-                params={"error": str(e)}
+                params={"error": str(e)},
             )
 
     async def aflush_db(self):
         """Drop the entire table"""
+
         async def _flush(db):
-            await db.execute('DROP TABLE IF EXISTS crawled_data')
+            await db.execute("DROP TABLE IF EXISTS crawled_data")
 
         try:
             await self.execute_with_retry(_flush)
@@ -380,42 +520,44 @@ class AsyncDatabaseManager:
                 message="Error flushing database: {error}",
                 tag="ERROR",
                 force_verbose=True,
-                params={"error": str(e)}
+                params={"error": str(e)},
             )
-            
-                
+
     async def _store_content(self, content: str, content_type: str) -> str:
         """Store content in filesystem and return hash"""
         if not content:
             return ""
-            
+
         content_hash = generate_content_hash(content)
         file_path = os.path.join(self.content_paths[content_type], content_hash)
-        
+
         # Only write if file doesn't exist
         if not os.path.exists(file_path):
-            async with aiofiles.open(file_path, 'w', encoding='utf-8') as f:
+            async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
                 await f.write(content)
-                
+
         return content_hash
 
-    async def _load_content(self, content_hash: str, content_type: str) -> Optional[str]:
+    async def _load_content(
+        self, content_hash: str, content_type: str
+    ) -> Optional[str]:
         """Load content from filesystem by hash"""
         if not content_hash:
             return None
-            
+
         file_path = os.path.join(self.content_paths[content_type], content_hash)
         try:
-            async with aiofiles.open(file_path, 'r', encoding='utf-8') as f:
+            async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
                 return await f.read()
         except:
             self.logger.error(
                 message="Failed to load content: {file_path}",
                 tag="ERROR",
                 force_verbose=True,
-                params={"file_path": file_path}
+                params={"file_path": file_path},
             )
             return None
 
+
 # Create a singleton instance
 async_db_manager = AsyncDatabaseManager()

crawl4ai/async_dispatcher.py

@@ -0,0 +1,647 @@
+from typing import Dict, Optional, List, Tuple
+from .async_configs import CrawlerRunConfig
+from .models import (
+    CrawlResult,
+    CrawlerTaskResult,
+    CrawlStatus,
+    DisplayMode,
+    CrawlStats,
+    DomainState,
+)
+
+from rich.live import Live
+from rich.table import Table
+from rich.console import Console
+from rich import box
+from datetime import timedelta
+from collections.abc import AsyncGenerator
+import time
+import psutil
+import asyncio
+import uuid
+
+from urllib.parse import urlparse
+import random
+from abc import ABC, abstractmethod
+
+from math import inf as infinity
+
+
+class RateLimiter:
+    def __init__(
+        self,
+        base_delay: Tuple[float, float] = (1.0, 3.0),
+        max_delay: float = 60.0,
+        max_retries: int = 3,
+        rate_limit_codes: List[int] = None,
+    ):
+        self.base_delay = base_delay
+        self.max_delay = max_delay
+        self.max_retries = max_retries
+        self.rate_limit_codes = rate_limit_codes or [429, 503]
+        self.domains: Dict[str, DomainState] = {}
+
+    def get_domain(self, url: str) -> str:
+        return urlparse(url).netloc
+
+    async def wait_if_needed(self, url: str) -> None:
+        domain = self.get_domain(url)
+        state = self.domains.get(domain)
+
+        if not state:
+            self.domains[domain] = DomainState()
+            state = self.domains[domain]
+
+        now = time.time()
+        if state.last_request_time:
+            wait_time = max(0, state.current_delay - (now - state.last_request_time))
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+
+        # Random delay within base range if no current delay
+        if state.current_delay == 0:
+            state.current_delay = random.uniform(*self.base_delay)
+
+        state.last_request_time = time.time()
+
+    def update_delay(self, url: str, status_code: int) -> bool:
+        domain = self.get_domain(url)
+        state = self.domains[domain]
+
+        if status_code in self.rate_limit_codes:
+            state.fail_count += 1
+            if state.fail_count > self.max_retries:
+                return False
+
+            # Exponential backoff with random jitter
+            state.current_delay = min(
+                state.current_delay * 2 * random.uniform(0.75, 1.25), self.max_delay
+            )
+        else:
+            # Gradually reduce delay on success
+            state.current_delay = max(
+                random.uniform(*self.base_delay), state.current_delay * 0.75
+            )
+            state.fail_count = 0
+
+        return True
+
+
+class CrawlerMonitor:
+    def __init__(
+        self,
+        max_visible_rows: int = 15,
+        display_mode: DisplayMode = DisplayMode.DETAILED,
+    ):
+        self.console = Console()
+        self.max_visible_rows = max_visible_rows
+        self.display_mode = display_mode
+        self.stats: Dict[str, CrawlStats] = {}
+        self.process = psutil.Process()
+        self.start_time = time.time()
+        self.live = Live(self._create_table(), refresh_per_second=2)
+
+    def start(self):
+        self.live.start()
+
+    def stop(self):
+        self.live.stop()
+
+    def add_task(self, task_id: str, url: str):
+        self.stats[task_id] = CrawlStats(
+            task_id=task_id, url=url, status=CrawlStatus.QUEUED
+        )
+        self.live.update(self._create_table())
+
+    def update_task(self, task_id: str, **kwargs):
+        if task_id in self.stats:
+            for key, value in kwargs.items():
+                setattr(self.stats[task_id], key, value)
+            self.live.update(self._create_table())
+
+    def _create_aggregated_table(self) -> Table:
+        """Creates a compact table showing only aggregated statistics"""
+        table = Table(
+            box=box.ROUNDED,
+            title="Crawler Status Overview",
+            title_style="bold magenta",
+            header_style="bold blue",
+            show_lines=True,
+        )
+
+        # Calculate statistics
+        total_tasks = len(self.stats)
+        queued = sum(
+            1 for stat in self.stats.values() if stat.status == CrawlStatus.QUEUED
+        )
+        in_progress = sum(
+            1 for stat in self.stats.values() if stat.status == CrawlStatus.IN_PROGRESS
+        )
+        completed = sum(
+            1 for stat in self.stats.values() if stat.status == CrawlStatus.COMPLETED
+        )
+        failed = sum(
+            1 for stat in self.stats.values() if stat.status == CrawlStatus.FAILED
+        )
+
+        # Memory statistics
+        current_memory = self.process.memory_info().rss / (1024 * 1024)
+        total_task_memory = sum(stat.memory_usage for stat in self.stats.values())
+        peak_memory = max(
+            (stat.peak_memory for stat in self.stats.values()), default=0.0
+        )
+
+        # Duration
+        duration = time.time() - self.start_time
+
+        # Create status row
+        table.add_column("Status", style="bold cyan")
+        table.add_column("Count", justify="right")
+        table.add_column("Percentage", justify="right")
+
+        table.add_row("Total Tasks", str(total_tasks), "100%")
+        table.add_row(
+            "[yellow]In Queue[/yellow]",
+            str(queued),
+            f"{(queued / total_tasks * 100):.1f}%" if total_tasks > 0 else "0%",
+        )
+        table.add_row(
+            "[blue]In Progress[/blue]",
+            str(in_progress),
+            f"{(in_progress / total_tasks * 100):.1f}%" if total_tasks > 0 else "0%",
+        )
+        table.add_row(
+            "[green]Completed[/green]",
+            str(completed),
+            f"{(completed / total_tasks * 100):.1f}%" if total_tasks > 0 else "0%",
+        )
+        table.add_row(
+            "[red]Failed[/red]",
+            str(failed),
+            f"{(failed / total_tasks * 100):.1f}%" if total_tasks > 0 else "0%",
+        )
+
+        # Add memory information
+        table.add_section()
+        table.add_row(
+            "[magenta]Current Memory[/magenta]", f"{current_memory:.1f} MB", ""
+        )
+        table.add_row(
+            "[magenta]Total Task Memory[/magenta]", f"{total_task_memory:.1f} MB", ""
+        )
+        table.add_row(
+            "[magenta]Peak Task Memory[/magenta]", f"{peak_memory:.1f} MB", ""
+        )
+        table.add_row(
+            "[yellow]Runtime[/yellow]",
+            str(timedelta(seconds=int(duration))),
+            "",
+        )
+
+        return table
+
+    def _create_detailed_table(self) -> Table:
+        table = Table(
+            box=box.ROUNDED,
+            title="Crawler Performance Monitor",
+            title_style="bold magenta",
+            header_style="bold blue",
+        )
+
+        # Add columns
+        table.add_column("Task ID", style="cyan", no_wrap=True)
+        table.add_column("URL", style="cyan", no_wrap=True)
+        table.add_column("Status", style="bold")
+        table.add_column("Memory (MB)", justify="right")
+        table.add_column("Peak (MB)", justify="right")
+        table.add_column("Duration", justify="right")
+        table.add_column("Info", style="italic")
+
+        # Add summary row
+        total_memory = sum(stat.memory_usage for stat in self.stats.values())
+        active_count = sum(
+            1 for stat in self.stats.values() if stat.status == CrawlStatus.IN_PROGRESS
+        )
+        completed_count = sum(
+            1 for stat in self.stats.values() if stat.status == CrawlStatus.COMPLETED
+        )
+        failed_count = sum(
+            1 for stat in self.stats.values() if stat.status == CrawlStatus.FAILED
+        )
+
+        table.add_row(
+            "[bold yellow]SUMMARY",
+            f"Total: {len(self.stats)}",
+            f"Active: {active_count}",
+            f"{total_memory:.1f}",
+            f"{self.process.memory_info().rss / (1024 * 1024):.1f}",
+            str(
+                timedelta(
+                    seconds=int(time.time() - self.start_time)
+                )
+            ),
+            f"✓{completed_count} ✗{failed_count}",
+            style="bold",
+        )
+
+        table.add_section()
+
+        # Add rows for each task
+        visible_stats = sorted(
+            self.stats.values(),
+            key=lambda x: (
+                x.status != CrawlStatus.IN_PROGRESS,
+                x.status != CrawlStatus.QUEUED,
+                x.end_time or infinity,
+            ),
+        )[: self.max_visible_rows]
+
+        for stat in visible_stats:
+            status_style = {
+                CrawlStatus.QUEUED: "white",
+                CrawlStatus.IN_PROGRESS: "yellow",
+                CrawlStatus.COMPLETED: "green",
+                CrawlStatus.FAILED: "red",
+            }[stat.status]
+
+            table.add_row(
+                stat.task_id[:8],  # Show first 8 chars of task ID
+                stat.url[:40] + "..." if len(stat.url) > 40 else stat.url,
+                f"[{status_style}]{stat.status.value}[/{status_style}]",
+                f"{stat.memory_usage:.1f}",
+                f"{stat.peak_memory:.1f}",
+                stat.duration,
+                stat.error_message[:40] if stat.error_message else "",
+            )
+
+        return table
+
+    def _create_table(self) -> Table:
+        """Creates the appropriate table based on display mode"""
+        if self.display_mode == DisplayMode.AGGREGATED:
+            return self._create_aggregated_table()
+        return self._create_detailed_table()
+
+
+class BaseDispatcher(ABC):
+    def __init__(
+        self,
+        rate_limiter: Optional[RateLimiter] = None,
+        monitor: Optional[CrawlerMonitor] = None,
+    ):
+        self.crawler = None
+        self._domain_last_hit: Dict[str, float] = {}
+        self.concurrent_sessions = 0
+        self.rate_limiter = rate_limiter
+        self.monitor = monitor
+
+    @abstractmethod
+    async def crawl_url(
+        self,
+        url: str,
+        config: CrawlerRunConfig,
+        task_id: str,
+        monitor: Optional[CrawlerMonitor] = None,
+    ) -> CrawlerTaskResult:
+        pass
+
+    @abstractmethod
+    async def run_urls(
+        self,
+        urls: List[str],
+        crawler: "AsyncWebCrawler",  # noqa: F821
+        config: CrawlerRunConfig,
+        monitor: Optional[CrawlerMonitor] = None,
+    ) -> List[CrawlerTaskResult]:
+        pass
+
+
+class MemoryAdaptiveDispatcher(BaseDispatcher):
+    def __init__(
+        self,
+        memory_threshold_percent: float = 90.0,
+        check_interval: float = 1.0,
+        max_session_permit: int = 20,
+        memory_wait_timeout: float = 300.0,  # 5 minutes default timeout
+        rate_limiter: Optional[RateLimiter] = None,
+        monitor: Optional[CrawlerMonitor] = None,
+    ):
+        super().__init__(rate_limiter, monitor)
+        self.memory_threshold_percent = memory_threshold_percent
+        self.check_interval = check_interval
+        self.max_session_permit = max_session_permit
+        self.memory_wait_timeout = memory_wait_timeout
+        self.result_queue = asyncio.Queue()  # Queue for storing results
+
+    async def crawl_url(
+        self,
+        url: str,
+        config: CrawlerRunConfig,
+        task_id: str,
+    ) -> CrawlerTaskResult:
+        start_time = time.time()
+        error_message = ""
+        memory_usage = peak_memory = 0.0
+
+        try:
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id, status=CrawlStatus.IN_PROGRESS, start_time=start_time
+                )
+            self.concurrent_sessions += 1
+
+            if self.rate_limiter:
+                await self.rate_limiter.wait_if_needed(url)
+
+            process = psutil.Process()
+            start_memory = process.memory_info().rss / (1024 * 1024)
+            result = await self.crawler.arun(url, config=config, session_id=task_id)
+            end_memory = process.memory_info().rss / (1024 * 1024)
+
+            memory_usage = peak_memory = end_memory - start_memory
+
+            if self.rate_limiter and result.status_code:
+                if not self.rate_limiter.update_delay(url, result.status_code):
+                    error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
+                    if self.monitor:
+                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                    result = CrawlerTaskResult(
+                        task_id=task_id,
+                        url=url,
+                        result=result,
+                        memory_usage=memory_usage,
+                        peak_memory=peak_memory,
+                        start_time=start_time,
+                        end_time=time.time(),
+                        error_message=error_message,
+                    )
+                    await self.result_queue.put(result)
+                    return result
+
+            if not result.success:
+                error_message = result.error_message
+                if self.monitor:
+                    self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+            elif self.monitor:
+                self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
+
+        except Exception as e:
+            error_message = str(e)
+            if self.monitor:
+                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+            result = CrawlResult(
+                url=url, html="", metadata={}, success=False, error_message=str(e)
+            )
+
+        finally:
+            end_time = time.time()
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id,
+                    end_time=end_time,
+                    memory_usage=memory_usage,
+                    peak_memory=peak_memory,
+                    error_message=error_message,
+                )
+            self.concurrent_sessions -= 1
+
+        return CrawlerTaskResult(
+            task_id=task_id,
+            url=url,
+            result=result,
+            memory_usage=memory_usage,
+            peak_memory=peak_memory,
+            start_time=start_time,
+            end_time=end_time,
+            error_message=error_message,
+        )
+
+    async def run_urls(
+        self,
+        urls: List[str],
+        crawler: "AsyncWebCrawler",  # noqa: F821
+        config: CrawlerRunConfig,
+    ) -> List[CrawlerTaskResult]:
+        self.crawler = crawler
+
+        if self.monitor:
+            self.monitor.start()
+
+        try:
+            pending_tasks = []
+            active_tasks = []
+            task_queue = []
+
+            for url in urls:
+                task_id = str(uuid.uuid4())
+                if self.monitor:
+                    self.monitor.add_task(task_id, url)
+                task_queue.append((url, task_id))
+
+            while task_queue or active_tasks:
+                wait_start_time = time.time()
+                while len(active_tasks) < self.max_session_permit and task_queue:
+                    if psutil.virtual_memory().percent >= self.memory_threshold_percent:
+                        # Check if we've exceeded the timeout
+                        if time.time() - wait_start_time > self.memory_wait_timeout:
+                            raise MemoryError(
+                                f"Memory usage above threshold ({self.memory_threshold_percent}%) for more than {self.memory_wait_timeout} seconds"
+                            )
+                        await asyncio.sleep(self.check_interval)
+                        continue
+
+                    url, task_id = task_queue.pop(0)
+                    task = asyncio.create_task(self.crawl_url(url, config, task_id))
+                    active_tasks.append(task)
+
+                if not active_tasks:
+                    await asyncio.sleep(self.check_interval)
+                    continue
+
+                done, pending = await asyncio.wait(
+                    active_tasks, return_when=asyncio.FIRST_COMPLETED
+                )
+
+                pending_tasks.extend(done)
+                active_tasks = list(pending)
+
+            return await asyncio.gather(*pending_tasks)
+        finally:
+            if self.monitor:
+                self.monitor.stop()
+
+    async def run_urls_stream(
+        self,
+        urls: List[str],
+        crawler: "AsyncWebCrawler", # noqa: F821
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlerTaskResult, None]:
+        self.crawler = crawler
+        if self.monitor:
+            self.monitor.start()
+
+        try:
+            active_tasks = []
+            task_queue = []
+            completed_count = 0
+            total_urls = len(urls)
+
+            # Initialize task queue
+            for url in urls:
+                task_id = str(uuid.uuid4())
+                if self.monitor:
+                    self.monitor.add_task(task_id, url)
+                task_queue.append((url, task_id))
+
+            while completed_count < total_urls:
+                # Start new tasks if memory permits
+                while len(active_tasks) < self.max_session_permit and task_queue:
+                    if psutil.virtual_memory().percent >= self.memory_threshold_percent:
+                        await asyncio.sleep(self.check_interval)
+                        continue
+
+                    url, task_id = task_queue.pop(0)
+                    task = asyncio.create_task(self.crawl_url(url, config, task_id))
+                    active_tasks.append(task)
+
+                if not active_tasks and not task_queue:
+                    break
+
+                # Wait for any task to complete and yield results
+                if active_tasks:
+                    done, pending = await asyncio.wait(
+                        active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
+                    )
+                    for completed_task in done:
+                        result = await completed_task
+                        completed_count += 1
+                        yield result
+                    active_tasks = list(pending)
+                else:
+                    await asyncio.sleep(self.check_interval)
+
+        finally:
+            if self.monitor:
+                self.monitor.stop()
+
+
+class SemaphoreDispatcher(BaseDispatcher):
+    def __init__(
+        self,
+        semaphore_count: int = 5,
+        max_session_permit: int = 20,
+        rate_limiter: Optional[RateLimiter] = None,
+        monitor: Optional[CrawlerMonitor] = None,
+    ):
+        super().__init__(rate_limiter, monitor)
+        self.semaphore_count = semaphore_count
+        self.max_session_permit = max_session_permit
+
+    async def crawl_url(
+        self,
+        url: str,
+        config: CrawlerRunConfig,
+        task_id: str,
+        semaphore: asyncio.Semaphore = None,
+    ) -> CrawlerTaskResult:
+        start_time = time.time()
+        error_message = ""
+        memory_usage = peak_memory = 0.0
+
+        try:
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id, status=CrawlStatus.IN_PROGRESS, start_time=start_time
+                )
+
+            if self.rate_limiter:
+                await self.rate_limiter.wait_if_needed(url)
+
+            async with semaphore:
+                process = psutil.Process()
+                start_memory = process.memory_info().rss / (1024 * 1024)
+                result = await self.crawler.arun(url, config=config, session_id=task_id)
+                end_memory = process.memory_info().rss / (1024 * 1024)
+
+                memory_usage = peak_memory = end_memory - start_memory
+
+                if self.rate_limiter and result.status_code:
+                    if not self.rate_limiter.update_delay(url, result.status_code):
+                        error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
+                        if self.monitor:
+                            self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                        return CrawlerTaskResult(
+                            task_id=task_id,
+                            url=url,
+                            result=result,
+                            memory_usage=memory_usage,
+                            peak_memory=peak_memory,
+                            start_time=start_time,
+                            end_time=time.time(),
+                            error_message=error_message,
+                        )
+
+                if not result.success:
+                    error_message = result.error_message
+                    if self.monitor:
+                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+                elif self.monitor:
+                    self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
+
+        except Exception as e:
+            error_message = str(e)
+            if self.monitor:
+                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
+            result = CrawlResult(
+                url=url, html="", metadata={}, success=False, error_message=str(e)
+            )
+
+        finally:
+            end_time = time.time()
+            if self.monitor:
+                self.monitor.update_task(
+                    task_id,
+                    end_time=end_time,
+                    memory_usage=memory_usage,
+                    peak_memory=peak_memory,
+                    error_message=error_message,
+                )
+
+        return CrawlerTaskResult(
+            task_id=task_id,
+            url=url,
+            result=result,
+            memory_usage=memory_usage,
+            peak_memory=peak_memory,
+            start_time=start_time,
+            end_time=end_time,
+            error_message=error_message,
+        )
+
+    async def run_urls(
+        self,
+        crawler: "AsyncWebCrawler",  # noqa: F821
+        urls: List[str],
+        config: CrawlerRunConfig,
+    ) -> List[CrawlerTaskResult]:
+        self.crawler = crawler
+        if self.monitor:
+            self.monitor.start()
+
+        try:
+            semaphore = asyncio.Semaphore(self.semaphore_count)
+            tasks = []
+
+            for url in urls:
+                task_id = str(uuid.uuid4())
+                if self.monitor:
+                    self.monitor.add_task(task_id, url)
+                task = asyncio.create_task(
+                    self.crawl_url(url, config, task_id, semaphore)
+                )
+                tasks.append(task)
+
+            return await asyncio.gather(*tasks, return_exceptions=True)
+        finally:
+            if self.monitor:
+                self.monitor.stop()

crawl4ai/async_logger.py

@@ -1,10 +1,11 @@
+from abc import ABC, abstractmethod
 from enum import Enum
-from typing import Optional, Dict, Any, Union
-from colorama import Fore, Back, Style, init
-import time
+from typing import Optional, Dict, Any
+from colorama import Fore, Style, init
 import os
 from datetime import datetime
 
+
 class LogLevel(Enum):
     DEBUG = 1
     INFO = 2
@@ -12,23 +13,54 @@ class LogLevel(Enum):
     WARNING = 4
     ERROR = 5
 
-class AsyncLogger:
+
+
+class AsyncLoggerBase(ABC):
+    @abstractmethod
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        pass
+
+    @abstractmethod
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        pass
+
+    @abstractmethod
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        pass
+
+    @abstractmethod
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        pass
+
+    @abstractmethod
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        pass
+
+    @abstractmethod
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+        pass
+
+    @abstractmethod
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+        pass
+
+class AsyncLogger(AsyncLoggerBase):
     """
     Asynchronous logger with support for colored console output and file logging.
     Supports templated messages with colored components.
     """
-    
+
     DEFAULT_ICONS = {
-        'INIT': '→',
-        'READY': '✓',
-        'FETCH': '↓',
-        'SCRAPE': '◆',
-        'EXTRACT': '■',
-        'COMPLETE': '●',
-        'ERROR': '×',
-        'DEBUG': '⋯',
-        'INFO': 'ℹ',
-        'WARNING': '⚠',
+        "INIT": "→",
+        "READY": "✓",
+        "FETCH": "↓",
+        "SCRAPE": "◆",
+        "EXTRACT": "■",
+        "COMPLETE": "●",
+        "ERROR": "×",
+        "DEBUG": "⋯",
+        "INFO": "ℹ",
+        "WARNING": "⚠",
     }
 
     DEFAULT_COLORS = {
@@ -42,15 +74,15 @@ class AsyncLogger:
     def __init__(
         self,
         log_file: Optional[str] = None,
-        log_level: LogLevel = LogLevel.INFO,
+        log_level: LogLevel = LogLevel.DEBUG,
         tag_width: int = 10,
         icons: Optional[Dict[str, str]] = None,
         colors: Optional[Dict[LogLevel, str]] = None,
-        verbose: bool = True
+        verbose: bool = True,
     ):
         """
         Initialize the logger.
-        
+
         Args:
             log_file: Optional file path for logging
             log_level: Minimum log level to display
@@ -66,7 +98,7 @@ class AsyncLogger:
         self.icons = icons or self.DEFAULT_ICONS
         self.colors = colors or self.DEFAULT_COLORS
         self.verbose = verbose
-        
+
         # Create log file directory if needed
         if log_file:
             os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
@@ -77,18 +109,20 @@ class AsyncLogger:
 
     def _get_icon(self, tag: str) -> str:
         """Get the icon for a tag, defaulting to info icon if not found."""
-        return self.icons.get(tag, self.icons['INFO'])
+        return self.icons.get(tag, self.icons["INFO"])
 
     def _write_to_file(self, message: str):
         """Write a message to the log file if configured."""
         if self.log_file:
-            timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
-            with open(self.log_file, 'a', encoding='utf-8') as f:
+            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+            with open(self.log_file, "a", encoding="utf-8") as f:
                 # Strip ANSI color codes for file output
-                clean_message = message.replace(Fore.RESET, '').replace(Style.RESET_ALL, '')
+                clean_message = message.replace(Fore.RESET, "").replace(
+                    Style.RESET_ALL, ""
+                )
                 for color in vars(Fore).values():
                     if isinstance(color, str):
-                        clean_message = clean_message.replace(color, '')
+                        clean_message = clean_message.replace(color, "")
                 f.write(f"[{timestamp}] {clean_message}\n")
 
     def _log(
@@ -99,11 +133,11 @@ class AsyncLogger:
         params: Optional[Dict[str, Any]] = None,
         colors: Optional[Dict[str, str]] = None,
         base_color: Optional[str] = None,
-        **kwargs
+        **kwargs,
     ):
         """
         Core logging method that handles message formatting and output.
-        
+
         Args:
             level: Log level for this message
             message: Message template string
@@ -120,7 +154,7 @@ class AsyncLogger:
             try:
                 # First format the message with raw parameters
                 formatted_message = message.format(**params)
-                
+
                 # Then apply colors if specified
                 if colors:
                     for key, color in colors.items():
@@ -128,12 +162,13 @@ class AsyncLogger:
                         if key in params:
                             value_str = str(params[key])
                             formatted_message = formatted_message.replace(
-                                value_str, 
-                                f"{color}{value_str}{Style.RESET_ALL}"
+                                value_str, f"{color}{value_str}{Style.RESET_ALL}"
                             )
-                            
+
             except KeyError as e:
-                formatted_message = f"LOGGING ERROR: Missing parameter {e} in message template"
+                formatted_message = (
+                    f"LOGGING ERROR: Missing parameter {e} in message template"
+                )
                 level = LogLevel.ERROR
         else:
             formatted_message = message
@@ -175,11 +210,11 @@ class AsyncLogger:
         success: bool,
         timing: float,
         tag: str = "FETCH",
-        url_length: int = 50
+        url_length: int = 50,
     ):
         """
         Convenience method for logging URL fetch status.
-        
+
         Args:
             url: The URL being processed
             success: Whether the operation was successful
@@ -195,24 +230,20 @@ class AsyncLogger:
                 "url": url,
                 "url_length": url_length,
                 "status": success,
-                "timing": timing
+                "timing": timing,
             },
             colors={
                 "status": Fore.GREEN if success else Fore.RED,
-                "timing": Fore.YELLOW
-            }
+                "timing": Fore.YELLOW,
+            },
         )
 
     def error_status(
-        self,
-        url: str,
-        error: str,
-        tag: str = "ERROR",
-        url_length: int = 50
+        self, url: str, error: str, tag: str = "ERROR", url_length: int = 50
     ):
         """
         Convenience method for logging error status.
-        
+
         Args:
             url: The URL being processed
             error: Error message
@@ -223,9 +254,57 @@ class AsyncLogger:
             level=LogLevel.ERROR,
             message="{url:.{url_length}}... | Error: {error}",
             tag=tag,
-            params={
-                "url": url,
-                "url_length": url_length,
-                "error": error
-            }
-        )
+            params={"url": url, "url_length": url_length, "error": error},
+        )
+
+class AsyncFileLogger(AsyncLoggerBase):
+    """
+    File-only asynchronous logger that writes logs to a specified file.
+    """
+
+    def __init__(self, log_file: str):
+        """
+        Initialize the file logger.
+
+        Args:
+            log_file: File path for logging
+        """
+        self.log_file = log_file
+        os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
+
+    def _write_to_file(self, level: str, message: str, tag: str):
+        """Write a message to the log file."""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+        with open(self.log_file, "a", encoding="utf-8") as f:
+            f.write(f"[{timestamp}] [{level}] [{tag}] {message}\n")
+
+    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
+        """Log a debug message to file."""
+        self._write_to_file("DEBUG", message, tag)
+
+    def info(self, message: str, tag: str = "INFO", **kwargs):
+        """Log an info message to file."""
+        self._write_to_file("INFO", message, tag)
+
+    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
+        """Log a success message to file."""
+        self._write_to_file("SUCCESS", message, tag)
+
+    def warning(self, message: str, tag: str = "WARNING", **kwargs):
+        """Log a warning message to file."""
+        self._write_to_file("WARNING", message, tag)
+
+    def error(self, message: str, tag: str = "ERROR", **kwargs):
+        """Log an error message to file."""
+        self._write_to_file("ERROR", message, tag)
+
+    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 50):
+        """Log URL fetch status to file."""
+        status = "SUCCESS" if success else "FAILED"
+        message = f"{url[:url_length]}... | Status: {status} | Time: {timing:.2f}s"
+        self._write_to_file("URL_STATUS", message, tag)
+
+    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 50):
+        """Log error status to file."""
+        message = f"{url[:url_length]}... | Error: {error}"
+        self._write_to_file("ERROR", message, tag)

crawl4ai/browser_manager.py

@@ -0,0 +1,873 @@
+import asyncio
+import time
+from typing import List, Optional
+import os
+import sys
+import shutil
+import tempfile
+import subprocess
+from playwright.async_api import BrowserContext
+import hashlib
+from .js_snippet import load_js_script
+from .config import DOWNLOAD_PAGE_TIMEOUT
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from playwright_stealth import StealthConfig
+from .utils import get_chromium_path
+
+stealth_config = StealthConfig(
+    webdriver=True,
+    chrome_app=True,
+    chrome_csi=True,
+    chrome_load_times=True,
+    chrome_runtime=True,
+    navigator_languages=True,
+    navigator_plugins=True,
+    navigator_permissions=True,
+    webgl_vendor=True,
+    outerdimensions=True,
+    navigator_hardware_concurrency=True,
+    media_codecs=True,
+)
+
+BROWSER_DISABLE_OPTIONS = [
+    "--disable-background-networking",
+    "--disable-background-timer-throttling",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-breakpad",
+    "--disable-client-side-phishing-detection",
+    "--disable-component-extensions-with-background-pages",
+    "--disable-default-apps",
+    "--disable-extensions",
+    "--disable-features=TranslateUI",
+    "--disable-hang-monitor",
+    "--disable-ipc-flooding-protection",
+    "--disable-popup-blocking",
+    "--disable-prompt-on-repost",
+    "--disable-sync",
+    "--force-color-profile=srgb",
+    "--metrics-recording-only",
+    "--no-first-run",
+    "--password-store=basic",
+    "--use-mock-keychain",
+]
+
+
+class ManagedBrowser:
+    """
+    Manages the browser process and context. This class allows to connect to the browser using CDP protocol.
+
+    Attributes:
+        browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                            Default: "chromium".
+        user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                     temporary directory may be used. Default: None.
+        headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                         Default: True.
+        browser_process (subprocess.Popen): The process object for the browser.
+        temp_dir (str): Temporary directory for user data if not provided.
+        debugging_port (int): Port for debugging the browser.
+        host (str): Host for debugging the browser.
+
+        Methods:
+            start(): Starts the browser process and returns the CDP endpoint URL.
+            _get_browser_path(): Returns the browser executable path based on OS and browser type.
+            _get_browser_args(): Returns browser-specific command line arguments.
+            _get_user_data_dir(): Returns the user data directory path.
+            _cleanup(): Terminates the browser process and removes the temporary directory.
+            create_profile(): Static method to create a user profile by launching a browser for user interaction.
+    """
+
+    browser_type: str
+    user_data_dir: str
+    headless: bool
+    browser_process: subprocess.Popen
+    temp_dir: str
+    debugging_port: int
+    host: str
+
+    def __init__(
+        self,
+        browser_type: str = "chromium",
+        user_data_dir: Optional[str] = None,
+        headless: bool = False,
+        logger=None,
+        host: str = "localhost",
+        debugging_port: int = 9222,
+        cdp_url: Optional[str] = None, 
+    ):
+        """
+        Initialize the ManagedBrowser instance.
+
+        Args:
+            browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit".
+                                Default: "chromium".
+            user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a
+                                         temporary directory may be used. Default: None.
+            headless (bool): Whether to run the browser in headless mode (no visible GUI).
+                             Default: True.
+            logger (logging.Logger): Logger instance for logging messages. Default: None.
+            host (str): Host for debugging the browser. Default: "localhost".
+            debugging_port (int): Port for debugging the browser. Default: 9222.
+            cdp_url (str or None): CDP URL to connect to the browser. Default: None.
+        """
+        self.browser_type = browser_type
+        self.user_data_dir = user_data_dir
+        self.headless = headless
+        self.browser_process = None
+        self.temp_dir = None
+        self.debugging_port = debugging_port
+        self.host = host
+        self.logger = logger
+        self.shutting_down = False
+        self.cdp_url = cdp_url
+
+    async def start(self) -> str:
+        """
+        Starts the browser process or returns CDP endpoint URL.
+        If cdp_url is provided, returns it directly.
+        If user_data_dir is not provided for local browser, creates a temporary directory.
+        
+        Returns:
+            str: CDP endpoint URL
+        """
+        # If CDP URL provided, just return it
+        if self.cdp_url:
+            return self.cdp_url
+
+        # Create temp dir if needed
+        if not self.user_data_dir:
+            self.temp_dir = tempfile.mkdtemp(prefix="browser-profile-")
+            self.user_data_dir = self.temp_dir
+
+        # Get browser path and args based on OS and browser type
+        # browser_path = self._get_browser_path()
+        args = await self._get_browser_args()
+
+        # Start browser process
+        try:
+            self.browser_process = subprocess.Popen(
+                args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )
+            # Monitor browser process output for errors
+            asyncio.create_task(self._monitor_browser_process())
+            await asyncio.sleep(2)  # Give browser time to start
+            return f"http://{self.host}:{self.debugging_port}"
+        except Exception as e:
+            await self.cleanup()
+            raise Exception(f"Failed to start browser: {e}")
+
+    async def _monitor_browser_process(self):
+        """
+        Monitor the browser process for unexpected termination.
+
+        How it works:
+        1. Read stdout and stderr from the browser process.
+        2. If the process has terminated, log the error message and terminate the browser.
+        3. If the shutting_down flag is set, log the normal termination message.
+        4. If any other error occurs, log the error message.
+
+        Note: This method should be called in a separate task to avoid blocking the main event loop.
+        """
+        if self.browser_process:
+            try:
+                stdout, stderr = await asyncio.gather(
+                    asyncio.to_thread(self.browser_process.stdout.read),
+                    asyncio.to_thread(self.browser_process.stderr.read),
+                )
+
+                # Check shutting_down flag BEFORE logging anything
+                if self.browser_process.poll() is not None:
+                    if not self.shutting_down:
+                        self.logger.error(
+                            message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}",
+                            tag="ERROR",
+                            params={
+                                "code": self.browser_process.returncode,
+                                "stdout": stdout.decode(),
+                                "stderr": stderr.decode(),
+                            },
+                        )
+                        await self.cleanup()
+                    else:
+                        self.logger.info(
+                            message="Browser process terminated normally | Code: {code}",
+                            tag="INFO",
+                            params={"code": self.browser_process.returncode},
+                        )
+            except Exception as e:
+                if not self.shutting_down:
+                    self.logger.error(
+                        message="Error monitoring browser process: {error}",
+                        tag="ERROR",
+                        params={"error": str(e)},
+                    )
+
+    def _get_browser_path_WIP(self) -> str:
+        """Returns the browser executable path based on OS and browser type"""
+        if sys.platform == "darwin":  # macOS
+            paths = {
+                "chromium": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                "firefox": "/Applications/Firefox.app/Contents/MacOS/firefox",
+                "webkit": "/Applications/Safari.app/Contents/MacOS/Safari",
+            }
+        elif sys.platform == "win32":  # Windows
+            paths = {
+                "chromium": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+                "firefox": "C:\\Program Files\\Mozilla Firefox\\firefox.exe",
+                "webkit": None,  # WebKit not supported on Windows
+            }
+        else:  # Linux
+            paths = {
+                "chromium": "google-chrome",
+                "firefox": "firefox",
+                "webkit": None,  # WebKit not supported on Linux
+            }
+
+        return paths.get(self.browser_type)
+
+    async def _get_browser_path(self) -> str:
+        browser_path = await get_chromium_path(self.browser_type)
+        return browser_path
+
+    async def _get_browser_args(self) -> List[str]:
+        """Returns browser-specific command line arguments"""
+        base_args = [await self._get_browser_path()]
+
+        if self.browser_type == "chromium":
+            args = [
+                f"--remote-debugging-port={self.debugging_port}",
+                f"--user-data-dir={self.user_data_dir}",
+            ]
+            if self.headless:
+                args.append("--headless=new")
+        elif self.browser_type == "firefox":
+            args = [
+                "--remote-debugging-port",
+                str(self.debugging_port),
+                "--profile",
+                self.user_data_dir,
+            ]
+            if self.headless:
+                args.append("--headless")
+        else:
+            raise NotImplementedError(f"Browser type {self.browser_type} not supported")
+
+        return base_args + args
+
+    async def cleanup(self):
+        """Cleanup browser process and temporary directory"""
+        # Set shutting_down flag BEFORE any termination actions
+        self.shutting_down = True
+
+        if self.browser_process:
+            try:
+                self.browser_process.terminate()
+                # Wait for process to end gracefully
+                for _ in range(10):  # 10 attempts, 100ms each
+                    if self.browser_process.poll() is not None:
+                        break
+                    await asyncio.sleep(0.1)
+
+                # Force kill if still running
+                if self.browser_process.poll() is None:
+                    self.browser_process.kill()
+                    await asyncio.sleep(0.1)  # Brief wait for kill to take effect
+
+            except Exception as e:
+                self.logger.error(
+                    message="Error terminating browser: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)},
+                )
+
+        if self.temp_dir and os.path.exists(self.temp_dir):
+            try:
+                shutil.rmtree(self.temp_dir)
+            except Exception as e:
+                self.logger.error(
+                    message="Error removing temporary directory: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)},
+                )
+                
+    # These methods have been moved to BrowserProfiler class
+    @staticmethod
+    async def create_profile(browser_config=None, profile_name=None, logger=None):
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Creates a browser profile by launching a browser for interactive user setup
+        and waits until the user closes it. The profile is stored in a directory that
+        can be used later with BrowserConfig.user_data_dir.
+        
+        Please use BrowserProfiler.create_profile() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            profile_path = await profiler.create_profile(profile_name="my-login-profile")
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler(logger=logger)
+        return await profiler.create_profile(profile_name=profile_name, browser_config=browser_config)
+    
+    @staticmethod
+    def list_profiles():
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Lists all available browser profiles in the Crawl4AI profiles directory.
+        
+        Please use BrowserProfiler.list_profiles() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            profiles = profiler.list_profiles()
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler()
+        return profiler.list_profiles()
+        
+    @staticmethod
+    def delete_profile(profile_name_or_path):
+        """
+        This method has been moved to the BrowserProfiler class.
+        
+        Delete a browser profile by name or path.
+        
+        Please use BrowserProfiler.delete_profile() instead.
+        
+        Example:
+            ```python
+            from crawl4ai.browser_profiler import BrowserProfiler
+            
+            profiler = BrowserProfiler()
+            success = profiler.delete_profile("my-profile")
+            ```
+        """
+        from .browser_profiler import BrowserProfiler
+        
+        # Create a BrowserProfiler instance and delegate to it
+        profiler = BrowserProfiler()
+        return profiler.delete_profile(profile_name_or_path)
+
+
+
+
+class BrowserManager:
+    """
+    Manages the browser instance and context.
+
+    Attributes:
+        config (BrowserConfig): Configuration object containing all browser settings
+        logger: Logger instance for recording events and errors
+        browser (Browser): The browser instance
+        default_context (BrowserContext): The default browser context
+        managed_browser (ManagedBrowser): The managed browser instance
+        playwright (Playwright): The Playwright instance
+        sessions (dict): Dictionary to store session information
+        session_ttl (int): Session timeout in seconds
+    """
+    
+
+    def __init__(self, browser_config: BrowserConfig, logger=None):
+        """
+        Initialize the BrowserManager with a browser configuration.
+
+        Args:
+            browser_config (BrowserConfig): Configuration object containing all browser settings
+            logger: Logger instance for recording events and errors
+        """
+        self.config: BrowserConfig = browser_config
+        self.logger = logger
+
+        # Browser state
+        self.browser = None
+        self.default_context = None
+        self.managed_browser = None
+        self.playwright = None
+
+        # Session management
+        self.sessions = {}
+        self.session_ttl = 1800  # 30 minutes
+
+        # Keep track of contexts by a "config signature," so each unique config reuses a single context
+        self.contexts_by_config = {}
+        self._contexts_lock = asyncio.Lock() 
+
+        # Initialize ManagedBrowser if needed
+        if self.config.use_managed_browser:
+            self.managed_browser = ManagedBrowser(
+                browser_type=self.config.browser_type,
+                user_data_dir=self.config.user_data_dir,
+                headless=self.config.headless,
+                logger=self.logger,
+                debugging_port=self.config.debugging_port,
+                cdp_url=self.config.cdp_url,
+            )
+
+    async def start(self):
+        """
+        Start the browser instance and set up the default context.
+
+        How it works:
+        1. Check if Playwright is already initialized.
+        2. If not, initialize Playwright.
+        3. If managed browser is used, start it and connect to the CDP endpoint.
+        4. If managed browser is not used, launch the browser and set up the default context.
+
+        Note: This method should be called in a separate task to avoid blocking the main event loop.
+        """
+        if self.playwright is None:
+            from playwright.async_api import async_playwright
+
+            self.playwright = await async_playwright().start()
+
+        if self.config.use_managed_browser:
+            cdp_url = await self.managed_browser.start()
+            self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
+            contexts = self.browser.contexts
+            if contexts:
+                self.default_context = contexts[0]
+            else:
+                self.default_context = await self.create_browser_context()
+                # self.default_context = await self.browser.new_context(
+                #     viewport={
+                #         "width": self.config.viewport_width,
+                #         "height": self.config.viewport_height,
+                #     },
+                #     storage_state=self.config.storage_state,
+                #     user_agent=self.config.headers.get(
+                #         "User-Agent", self.config.user_agent
+                #     ),
+                #     accept_downloads=self.config.accept_downloads,
+                #     ignore_https_errors=self.config.ignore_https_errors,
+                #     java_script_enabled=self.config.java_script_enabled,
+                # )
+            await self.setup_context(self.default_context)
+        else:
+            browser_args = self._build_browser_args()
+
+            # Launch appropriate browser type
+            if self.config.browser_type == "firefox":
+                self.browser = await self.playwright.firefox.launch(**browser_args)
+            elif self.config.browser_type == "webkit":
+                self.browser = await self.playwright.webkit.launch(**browser_args)
+            else:
+                self.browser = await self.playwright.chromium.launch(**browser_args)
+
+            self.default_context = self.browser
+
+    def _build_browser_args(self) -> dict:
+        """Build browser launch arguments from config."""
+        args = [
+            "--disable-gpu",
+            "--disable-gpu-compositing",
+            "--disable-software-rasterizer",
+            "--no-sandbox",
+            "--disable-dev-shm-usage",
+            "--no-first-run",
+            "--no-default-browser-check",
+            "--disable-infobars",
+            "--window-position=0,0",
+            "--ignore-certificate-errors",
+            "--ignore-certificate-errors-spki-list",
+            "--disable-blink-features=AutomationControlled",
+            "--window-position=400,0",
+            "--disable-renderer-backgrounding",
+            "--disable-ipc-flooding-protection",
+            "--force-color-profile=srgb",
+            "--mute-audio",
+            "--disable-background-timer-throttling",
+            # "--single-process",
+            f"--window-size={self.config.viewport_width},{self.config.viewport_height}",
+        ]
+
+        if self.config.light_mode:
+            args.extend(BROWSER_DISABLE_OPTIONS)
+
+        if self.config.text_mode:
+            args.extend(
+                [
+                    "--blink-settings=imagesEnabled=false",
+                    "--disable-remote-fonts",
+                    "--disable-images",
+                    "--disable-javascript",
+                    "--disable-software-rasterizer",
+                    "--disable-dev-shm-usage",
+                ]
+            )
+
+        if self.config.extra_args:
+            args.extend(self.config.extra_args)
+
+        browser_args = {"headless": self.config.headless, "args": args}
+
+        if self.config.chrome_channel:
+            browser_args["channel"] = self.config.chrome_channel
+
+        if self.config.accept_downloads:
+            browser_args["downloads_path"] = self.config.downloads_path or os.path.join(
+                os.getcwd(), "downloads"
+            )
+            os.makedirs(browser_args["downloads_path"], exist_ok=True)
+
+        if self.config.proxy or self.config.proxy_config:
+            from playwright.async_api import ProxySettings
+
+            proxy_settings = (
+                ProxySettings(server=self.config.proxy)
+                if self.config.proxy
+                else ProxySettings(
+                    server=self.config.proxy_config.get("server"),
+                    username=self.config.proxy_config.get("username"),
+                    password=self.config.proxy_config.get("password"),
+                )
+            )
+            browser_args["proxy"] = proxy_settings
+
+        return browser_args
+
+    async def setup_context(
+        self,
+        context: BrowserContext,
+        crawlerRunConfig: CrawlerRunConfig = None,
+        is_default=False,
+    ):
+        """
+        Set up a browser context with the configured options.
+
+        How it works:
+        1. Set extra HTTP headers if provided.
+        2. Add cookies if provided.
+        3. Load storage state if provided.
+        4. Accept downloads if enabled.
+        5. Set default timeouts for navigation and download.
+        6. Set user agent if provided.
+        7. Set browser hints if provided.
+        8. Set proxy if provided.
+        9. Set downloads path if provided.
+        10. Set storage state if provided.
+        11. Set cache if provided.
+        12. Set extra HTTP headers if provided.
+        13. Add cookies if provided.
+        14. Set default timeouts for navigation and download if enabled.
+        15. Set user agent if provided.
+        16. Set browser hints if provided.
+
+        Args:
+            context (BrowserContext): The browser context to set up
+            crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings
+            is_default (bool): Flag indicating if this is the default context
+        Returns:
+            None
+        """
+        if self.config.headers:
+            await context.set_extra_http_headers(self.config.headers)
+
+        if self.config.cookies:
+            await context.add_cookies(self.config.cookies)
+
+        if self.config.storage_state:
+            await context.storage_state(path=None)
+
+        if self.config.accept_downloads:
+            context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT)
+            if self.config.downloads_path:
+                context._impl_obj._options["accept_downloads"] = True
+                context._impl_obj._options[
+                    "downloads_path"
+                ] = self.config.downloads_path
+
+        # Handle user agent and browser hints
+        if self.config.user_agent:
+            combined_headers = {
+                "User-Agent": self.config.user_agent,
+                "sec-ch-ua": self.config.browser_hint,
+            }
+            combined_headers.update(self.config.headers)
+            await context.set_extra_http_headers(combined_headers)
+
+        # Add default cookie
+        await context.add_cookies(
+            [
+                {
+                    "name": "cookiesEnabled",
+                    "value": "true",
+                    "url": crawlerRunConfig.url
+                    if crawlerRunConfig
+                    else "https://crawl4ai.com/",
+                }
+            ]
+        )
+
+        # Handle navigator overrides
+        if crawlerRunConfig:
+            if (
+                crawlerRunConfig.override_navigator
+                or crawlerRunConfig.simulate_user
+                or crawlerRunConfig.magic
+            ):
+                await context.add_init_script(load_js_script("navigator_overrider"))        
+
+    async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None):
+        """
+        Creates and returns a new browser context with configured settings.
+        Applies text-only mode settings if text_mode is enabled in config.
+
+        Returns:
+            Context: Browser context object with the specified configurations
+        """
+        # Base settings
+        user_agent = self.config.headers.get("User-Agent", self.config.user_agent) 
+        viewport_settings = {
+            "width": self.config.viewport_width,
+            "height": self.config.viewport_height,
+        }
+        proxy_settings = {"server": self.config.proxy} if self.config.proxy else None
+
+        blocked_extensions = [
+            # Images
+            "jpg",
+            "jpeg",
+            "png",
+            "gif",
+            "webp",
+            "svg",
+            "ico",
+            "bmp",
+            "tiff",
+            "psd",
+            # Fonts
+            "woff",
+            "woff2",
+            "ttf",
+            "otf",
+            "eot",
+            # Styles
+            # 'css', 'less', 'scss', 'sass',
+            # Media
+            "mp4",
+            "webm",
+            "ogg",
+            "avi",
+            "mov",
+            "wmv",
+            "flv",
+            "m4v",
+            "mp3",
+            "wav",
+            "aac",
+            "m4a",
+            "opus",
+            "flac",
+            # Documents
+            "pdf",
+            "doc",
+            "docx",
+            "xls",
+            "xlsx",
+            "ppt",
+            "pptx",
+            # Archives
+            "zip",
+            "rar",
+            "7z",
+            "tar",
+            "gz",
+            # Scripts and data
+            "xml",
+            "swf",
+            "wasm",
+        ]
+
+        # Common context settings
+        context_settings = {
+            "user_agent": user_agent,
+            "viewport": viewport_settings,
+            "proxy": proxy_settings,
+            "accept_downloads": self.config.accept_downloads,
+            "storage_state": self.config.storage_state,
+            "ignore_https_errors": self.config.ignore_https_errors,
+            "device_scale_factor": 1.0,
+            "java_script_enabled": self.config.java_script_enabled,
+        }
+        
+        if crawlerRunConfig:
+            # Check if there is value for crawlerRunConfig.proxy_config set add that to context
+            if crawlerRunConfig.proxy_config:
+                proxy_settings = {
+                    "server": crawlerRunConfig.proxy_config.server,
+                }
+                if crawlerRunConfig.proxy_config.username:
+                    proxy_settings.update({
+                        "username": crawlerRunConfig.proxy_config.username,
+                        "password": crawlerRunConfig.proxy_config.password,
+                    })
+                context_settings["proxy"] = proxy_settings
+
+        if self.config.text_mode:
+            text_mode_settings = {
+                "has_touch": False,
+                "is_mobile": False,
+            }
+            # Update context settings with text mode settings
+            context_settings.update(text_mode_settings)
+
+        # Create and return the context with all settings
+        context = await self.browser.new_context(**context_settings)
+
+        # Apply text mode settings if enabled
+        if self.config.text_mode:
+            # Create and apply route patterns for each extension
+            for ext in blocked_extensions:
+                await context.route(f"**/*.{ext}", lambda route: route.abort())
+        return context
+
+    def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str:
+        """
+        Converts the crawlerRunConfig into a dict, excludes ephemeral fields,
+        then returns a hash of the sorted JSON. This yields a stable signature
+        that identifies configurations requiring a unique browser context.
+        """
+        import json
+
+        config_dict = crawlerRunConfig.__dict__.copy()
+        # Exclude items that do not affect browser-level setup.
+        # Expand or adjust as needed, e.g. chunking_strategy is purely for data extraction, not for browser config.
+        ephemeral_keys = [
+            "session_id",
+            "js_code",
+            "scraping_strategy",
+            "extraction_strategy",
+            "chunking_strategy",
+            "cache_mode",
+            "content_filter",
+            "semaphore_count",
+            "url"
+        ]
+        for key in ephemeral_keys:
+            if key in config_dict:
+                del config_dict[key]
+        # Convert to canonical JSON string
+        signature_json = json.dumps(config_dict, sort_keys=True, default=str)
+
+        # Hash the JSON so we get a compact, unique string
+        signature_hash = hashlib.sha256(signature_json.encode("utf-8")).hexdigest()
+        return signature_hash
+
+    async def get_page(self, crawlerRunConfig: CrawlerRunConfig):
+        """
+        Get a page for the given session ID, creating a new one if needed.
+
+        Args:
+            crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings
+
+        Returns:
+            (page, context): The Page and its BrowserContext
+        """
+        self._cleanup_expired_sessions()
+
+        # If a session_id is provided and we already have it, reuse that page + context
+        if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions:
+            context, page, _ = self.sessions[crawlerRunConfig.session_id]
+            # Update last-used timestamp
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+            return page, context
+
+        # If using a managed browser, just grab the shared default_context
+        if self.config.use_managed_browser:
+            context = self.default_context
+            page = await context.new_page()
+        else:
+            # Otherwise, check if we have an existing context for this config
+            config_signature = self._make_config_signature(crawlerRunConfig)
+
+            async with self._contexts_lock:
+                if config_signature in self.contexts_by_config:
+                    context = self.contexts_by_config[config_signature]
+                else:
+                    # Create and setup a new context
+                    context = await self.create_browser_context(crawlerRunConfig)
+                    await self.setup_context(context, crawlerRunConfig)
+                    self.contexts_by_config[config_signature] = context
+
+            # Create a new page from the chosen context
+            page = await context.new_page()
+
+        # If a session_id is specified, store this session so we can reuse later
+        if crawlerRunConfig.session_id:
+            self.sessions[crawlerRunConfig.session_id] = (context, page, time.time())
+
+        return page, context
+
+    async def kill_session(self, session_id: str):
+        """
+        Kill a browser session and clean up resources.
+
+        Args:
+            session_id (str): The session ID to kill.
+        """
+        if session_id in self.sessions:
+            context, page, _ = self.sessions[session_id]
+            await page.close()
+            if not self.config.use_managed_browser:
+                await context.close()
+            del self.sessions[session_id]
+
+    def _cleanup_expired_sessions(self):
+        """Clean up expired sessions based on TTL."""
+        current_time = time.time()
+        expired_sessions = [
+            sid
+            for sid, (_, _, last_used) in self.sessions.items()
+            if current_time - last_used > self.session_ttl
+        ]
+        for sid in expired_sessions:
+            asyncio.create_task(self.kill_session(sid))
+
+    async def close(self):
+        """Close all browser resources and clean up."""
+        if self.config.sleep_on_close:
+            await asyncio.sleep(0.5)
+
+        session_ids = list(self.sessions.keys())
+        for session_id in session_ids:
+            await self.kill_session(session_id)
+
+        # Now close all contexts we created. This reclaims memory from ephemeral contexts.
+        for ctx in self.contexts_by_config.values():
+            try:
+                await ctx.close()
+            except Exception as e:
+                self.logger.error(
+                    message="Error closing context: {error}",
+                    tag="ERROR",
+                    params={"error": str(e)}
+                )
+        self.contexts_by_config.clear()
+
+        if self.browser:
+            await self.browser.close()
+            self.browser = None
+
+        if self.managed_browser:
+            await asyncio.sleep(0.5)
+            await self.managed_browser.cleanup()
+            self.managed_browser = None
+
+        if self.playwright:
+            await self.playwright.stop()
+            self.playwright = None

crawl4ai/browser_profiler.py

@@ -0,0 +1,544 @@
+"""
+Browser Profiler Module
+
+This module provides a dedicated class for managing browser profiles
+that can be used for identity-based crawling with Crawl4AI.
+"""
+
+import os
+import asyncio
+import signal
+import sys
+import datetime
+import uuid
+import shutil
+from typing import List, Dict, Optional, Any
+from colorama import Fore, Style, init
+
+from .async_configs import BrowserConfig
+from .browser_manager import ManagedBrowser
+from .async_logger import AsyncLogger, AsyncLoggerBase
+from .utils import get_home_folder
+
+
+class BrowserProfiler:
+    """
+    A dedicated class for managing browser profiles for Crawl4AI.
+    
+    The BrowserProfiler allows you to:
+    - Create browser profiles interactively
+    - List available profiles
+    - Delete profiles when no longer needed
+    - Get profile paths for use in BrowserConfig
+    
+    Profiles are stored by default in ~/.crawl4ai/profiles/
+    """
+    
+    def __init__(self, logger: Optional[AsyncLoggerBase] = None):
+        """
+        Initialize the BrowserProfiler.
+        
+        Args:
+            logger (AsyncLoggerBase, optional): Logger for outputting messages.
+                If None, a default AsyncLogger will be created.
+        """
+        # Initialize colorama for colorful terminal output
+        init()
+        
+        # Create a logger if not provided
+        if logger is None:
+            self.logger = AsyncLogger(verbose=True)
+        elif not isinstance(logger, AsyncLoggerBase):
+            self.logger = AsyncLogger(verbose=True)
+        else:
+            self.logger = logger
+            
+        # Ensure profiles directory exists
+        self.profiles_dir = os.path.join(get_home_folder(), "profiles")
+        os.makedirs(self.profiles_dir, exist_ok=True)
+    
+    async def create_profile(self, 
+                            profile_name: Optional[str] = None, 
+                            browser_config: Optional[BrowserConfig] = None) -> Optional[str]:
+        """
+        Creates a browser profile by launching a browser for interactive user setup
+        and waits until the user closes it. The profile is stored in a directory that
+        can be used later with BrowserConfig.user_data_dir.
+        
+        Args:
+            profile_name (str, optional): Name for the profile directory.
+                If None, a name is generated based on timestamp.
+            browser_config (BrowserConfig, optional): Configuration for the browser.
+                If None, a default configuration is used with headless=False.
+                
+        Returns:
+            str: Path to the created profile directory, or None if creation failed
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Create a profile interactively
+            profile_path = await profiler.create_profile(
+                profile_name="my-login-profile"
+            )
+            
+            # Use the profile in a crawler
+            browser_config = BrowserConfig(
+                headless=True,
+                use_managed_browser=True,
+                user_data_dir=profile_path
+            )
+            
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                # The crawler will now use your profile with all your cookies and login state
+                result = await crawler.arun("https://example.com/dashboard")
+            ```
+        """
+        # Create default browser config if none provided
+        if browser_config is None:
+            from .async_configs import BrowserConfig
+            browser_config = BrowserConfig(
+                browser_type="chromium",
+                headless=False,  # Must be visible for user interaction
+                verbose=True
+            )
+        else:
+            # Ensure headless is False for user interaction
+            browser_config.headless = False
+            
+        # Generate profile name if not provided
+        if not profile_name:
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            profile_name = f"profile_{timestamp}_{uuid.uuid4().hex[:6]}"
+            
+        # Sanitize profile name (replace spaces and special chars)
+        profile_name = "".join(c if c.isalnum() or c in "-_" else "_" for c in profile_name)
+        
+        # Set user data directory
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        os.makedirs(profile_path, exist_ok=True)
+        
+        # Print instructions for the user with colorama formatting
+        border = f"{Fore.CYAN}{'='*80}{Style.RESET_ALL}"
+        self.logger.info(f"\n{border}", tag="PROFILE")
+        self.logger.info(f"Creating browser profile: {Fore.GREEN}{profile_name}{Style.RESET_ALL}", tag="PROFILE")
+        self.logger.info(f"Profile directory: {Fore.YELLOW}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+        
+        self.logger.info("\nInstructions:", tag="PROFILE")
+        self.logger.info("1. A browser window will open for you to set up your profile.", tag="PROFILE")
+        self.logger.info(f"2. {Fore.CYAN}Log in to websites{Style.RESET_ALL}, configure settings, etc. as needed.", tag="PROFILE")
+        self.logger.info(f"3. When you're done, {Fore.YELLOW}press 'q' in this terminal{Style.RESET_ALL} to close the browser.", tag="PROFILE")
+        self.logger.info("4. The profile will be saved and ready to use with Crawl4AI.", tag="PROFILE")
+        self.logger.info(f"{border}\n", tag="PROFILE")
+        
+        # Create managed browser instance
+        managed_browser = ManagedBrowser(
+            browser_type=browser_config.browser_type,
+            user_data_dir=profile_path,
+            headless=False,  # Must be visible
+            logger=self.logger,
+            debugging_port=browser_config.debugging_port
+        )
+        
+        # Set up signal handlers to ensure cleanup on interrupt
+        original_sigint = signal.getsignal(signal.SIGINT)
+        original_sigterm = signal.getsignal(signal.SIGTERM)
+        
+        # Define cleanup handler for signals
+        async def cleanup_handler(sig, frame):
+            self.logger.warning("\nCleaning up browser process...", tag="PROFILE")
+            await managed_browser.cleanup()
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            if sig == signal.SIGINT:
+                self.logger.error("Profile creation interrupted. Profile may be incomplete.", tag="PROFILE")
+                sys.exit(1)
+                
+        # Set signal handlers
+        def sigint_handler(sig, frame):
+            asyncio.create_task(cleanup_handler(sig, frame))
+        
+        signal.signal(signal.SIGINT, sigint_handler)
+        signal.signal(signal.SIGTERM, sigint_handler)
+        
+        # Event to signal when user is done with the browser
+        user_done_event = asyncio.Event()
+        
+        # Run keyboard input loop in a separate task
+        async def listen_for_quit_command():
+            import termios
+            import tty
+            import select
+            
+            # First output the prompt
+            self.logger.info(f"{Fore.CYAN}Press '{Fore.WHITE}q{Fore.CYAN}' when you've finished using the browser...{Style.RESET_ALL}", tag="PROFILE")
+            
+            # Save original terminal settings
+            fd = sys.stdin.fileno()
+            old_settings = termios.tcgetattr(fd)
+            
+            try:
+                # Switch to non-canonical mode (no line buffering)
+                tty.setcbreak(fd)
+                
+                while True:
+                    # Check if input is available (non-blocking)
+                    readable, _, _ = select.select([sys.stdin], [], [], 0.5)
+                    if readable:
+                        key = sys.stdin.read(1)
+                        if key.lower() == 'q':
+                            self.logger.info(f"{Fore.GREEN}Closing browser and saving profile...{Style.RESET_ALL}", tag="PROFILE")
+                            user_done_event.set()
+                            return
+                    
+                    # Check if the browser process has already exited
+                    if managed_browser.browser_process and managed_browser.browser_process.poll() is not None:
+                        self.logger.info("Browser already closed. Ending input listener.", tag="PROFILE")
+                        user_done_event.set()
+                        return
+                        
+                    await asyncio.sleep(0.1)
+            
+            finally:
+                # Restore terminal settings 
+                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+        
+        try:
+            # Start the browser
+            await managed_browser.start()
+            
+            # Check if browser started successfully
+            browser_process = managed_browser.browser_process
+            if not browser_process:
+                self.logger.error("Failed to start browser process.", tag="PROFILE")
+                return None
+            
+            self.logger.info(f"Browser launched. {Fore.CYAN}Waiting for you to finish...{Style.RESET_ALL}", tag="PROFILE") 
+            
+            # Start listening for keyboard input
+            listener_task = asyncio.create_task(listen_for_quit_command())
+            
+            # Wait for either the user to press 'q' or for the browser process to exit naturally
+            while not user_done_event.is_set() and browser_process.poll() is None:
+                await asyncio.sleep(0.5)
+            
+            # Cancel the listener task if it's still running
+            if not listener_task.done():
+                listener_task.cancel()
+                try:
+                    await listener_task
+                except asyncio.CancelledError:
+                    pass
+            
+            # If the browser is still running and the user pressed 'q', terminate it
+            if browser_process.poll() is None and user_done_event.is_set():
+                self.logger.info("Terminating browser process...", tag="PROFILE")
+                await managed_browser.cleanup()
+            
+            self.logger.success(f"Browser closed. Profile saved at: {Fore.GREEN}{profile_path}{Style.RESET_ALL}", tag="PROFILE")
+                
+        except Exception as e:
+            self.logger.error(f"Error creating profile: {str(e)}", tag="PROFILE")
+            await managed_browser.cleanup()
+            return None
+        finally:
+            # Restore original signal handlers
+            signal.signal(signal.SIGINT, original_sigint)
+            signal.signal(signal.SIGTERM, original_sigterm)
+            
+            # Make sure browser is fully cleaned up
+            await managed_browser.cleanup()
+        
+        # Return the profile path
+        return profile_path
+    
+    def list_profiles(self) -> List[Dict[str, Any]]:
+        """
+        Lists all available browser profiles in the Crawl4AI profiles directory.
+        
+        Returns:
+            list: A list of dictionaries containing profile information:
+                  [{"name": "profile_name", "path": "/path/to/profile", "created": datetime, "type": "chromium|firefox"}]
+                  
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # List all available profiles
+            profiles = profiler.list_profiles()
+            
+            for profile in profiles:
+                print(f"Profile: {profile['name']}")
+                print(f"  Path: {profile['path']}")
+                print(f"  Created: {profile['created']}")
+                print(f"  Browser type: {profile['type']}")
+            ```
+        """
+        if not os.path.exists(self.profiles_dir):
+            return []
+            
+        profiles = []
+        
+        for name in os.listdir(self.profiles_dir):
+            profile_path = os.path.join(self.profiles_dir, name)
+            
+            # Skip if not a directory
+            if not os.path.isdir(profile_path):
+                continue
+                
+            # Check if this looks like a valid browser profile
+            # For Chromium: Look for Preferences file
+            # For Firefox: Look for prefs.js file
+            is_valid = False
+            
+            if os.path.exists(os.path.join(profile_path, "Preferences")) or \
+               os.path.exists(os.path.join(profile_path, "Default", "Preferences")):
+                is_valid = "chromium"
+            elif os.path.exists(os.path.join(profile_path, "prefs.js")):
+                is_valid = "firefox"
+                
+            if is_valid:
+                # Get creation time
+                created = datetime.datetime.fromtimestamp(
+                    os.path.getctime(profile_path)
+                )
+                
+                profiles.append({
+                    "name": name,
+                    "path": profile_path,
+                    "created": created,
+                    "type": is_valid
+                })
+                
+        # Sort by creation time, newest first
+        profiles.sort(key=lambda x: x["created"], reverse=True)
+        
+        return profiles
+    
+    def get_profile_path(self, profile_name: str) -> Optional[str]:
+        """
+        Get the full path to a profile by name.
+        
+        Args:
+            profile_name (str): Name of the profile (not the full path)
+            
+        Returns:
+            str: Full path to the profile directory, or None if not found
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            path = profiler.get_profile_path("my-profile")
+            if path:
+                print(f"Profile path: {path}")
+            else:
+                print("Profile not found")
+            ```
+        """
+        profile_path = os.path.join(self.profiles_dir, profile_name)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            return None
+        
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return None  # Not a valid browser profile
+            
+        return profile_path
+    
+    def delete_profile(self, profile_name_or_path: str) -> bool:
+        """
+        Delete a browser profile by name or path.
+        
+        Args:
+            profile_name_or_path (str): Name of the profile or full path to profile directory
+            
+        Returns:
+            bool: True if the profile was deleted successfully, False otherwise
+            
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Delete by name
+            success = profiler.delete_profile("my-profile")
+            
+            # Delete by path
+            success = profiler.delete_profile("/path/to/.crawl4ai/profiles/my-profile")
+            ```
+        """
+        # Determine if input is a name or a path
+        if os.path.isabs(profile_name_or_path):
+            # Full path provided
+            profile_path = profile_name_or_path
+        else:
+            # Just a name provided, construct path
+            profile_path = os.path.join(self.profiles_dir, profile_name_or_path)
+        
+        # Check if path exists and is a valid profile
+        if not os.path.isdir(profile_path):
+            return False
+            
+        # Look for profile indicators
+        is_profile = (
+            os.path.exists(os.path.join(profile_path, "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "Default", "Preferences")) or
+            os.path.exists(os.path.join(profile_path, "prefs.js"))
+        )
+        
+        if not is_profile:
+            return False  # Not a valid browser profile
+            
+        # Delete the profile directory
+        try:
+            shutil.rmtree(profile_path)
+            return True
+        except Exception:
+            return False
+            
+    async def interactive_manager(self, crawl_callback=None):
+        """
+        Launch an interactive profile management console.
+        
+        Args:
+            crawl_callback (callable, optional): Function to call when selecting option to use 
+                a profile for crawling. It will be called with (profile_path, url).
+                
+        Example:
+            ```python
+            profiler = BrowserProfiler()
+            
+            # Define a custom crawl function
+            async def my_crawl_function(profile_path, url):
+                print(f"Crawling {url} with profile {profile_path}")
+                # Implement your crawling logic here
+                
+            # Start interactive manager
+            await profiler.interactive_manager(crawl_callback=my_crawl_function)
+            ```
+        """
+        while True:
+            self.logger.info(f"\n{Fore.CYAN}Profile Management Options:{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"1. {Fore.GREEN}Create a new profile{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"2. {Fore.YELLOW}List available profiles{Style.RESET_ALL}", tag="MENU")
+            self.logger.info(f"3. {Fore.RED}Delete a profile{Style.RESET_ALL}", tag="MENU")
+            
+            # Only show crawl option if callback provided
+            if crawl_callback:
+                self.logger.info(f"4. {Fore.CYAN}Use a profile to crawl a website{Style.RESET_ALL}", tag="MENU")
+                self.logger.info(f"5. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "5"
+            else:
+                self.logger.info(f"4. {Fore.MAGENTA}Exit{Style.RESET_ALL}", tag="MENU")
+                exit_option = "4"
+            
+            choice = input(f"\n{Fore.CYAN}Enter your choice (1-{exit_option}): {Style.RESET_ALL}")
+            
+            if choice == "1":
+                # Create new profile
+                name = input(f"{Fore.GREEN}Enter a name for the new profile (or press Enter for auto-generated name): {Style.RESET_ALL}")
+                await self.create_profile(name or None)
+                
+            elif choice == "2":
+                # List profiles
+                profiles = self.list_profiles()
+                
+                if not profiles:
+                    self.logger.warning("  No profiles found. Create one first with option 1.", tag="PROFILES")
+                    continue
+                
+                # Print profile information with colorama formatting
+                self.logger.info("\nAvailable profiles:", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {Fore.CYAN}{profile['name']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Path: {Fore.YELLOW}{profile['path']}{Style.RESET_ALL}", tag="PROFILES")
+                    self.logger.info(f"    Created: {profile['created'].strftime('%Y-%m-%d %H:%M:%S')}", tag="PROFILES")
+                    self.logger.info(f"    Browser type: {profile['type']}", tag="PROFILES")
+                    self.logger.info("", tag="PROFILES")  # Empty line for spacing
+                
+            elif choice == "3":
+                # Delete profile
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found to delete", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to delete
+                profile_idx = input(f"{Fore.RED}Enter the number of the profile to delete (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_name = profiles[idx]["name"]
+                        self.logger.info(f"Deleting profile: {Fore.YELLOW}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                        
+                        # Confirm deletion
+                        confirm = input(f"{Fore.RED}Are you sure you want to delete this profile? (y/n): {Style.RESET_ALL}")
+                        if confirm.lower() == 'y':
+                            success = self.delete_profile(profiles[idx]["path"])
+                            
+                            if success:
+                                self.logger.success(f"Profile {Fore.GREEN}{profile_name}{Style.RESET_ALL} deleted successfully", tag="PROFILES")
+                            else:
+                                self.logger.error(f"Failed to delete profile {Fore.RED}{profile_name}{Style.RESET_ALL}", tag="PROFILES")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == "4" and crawl_callback:
+                # Use profile to crawl a site
+                profiles = self.list_profiles()
+                if not profiles:
+                    self.logger.warning("No profiles found. Create one first.", tag="PROFILES")
+                    continue
+                    
+                # Display numbered list
+                self.logger.info(f"\n{Fore.YELLOW}Available profiles:{Style.RESET_ALL}", tag="PROFILES")
+                for i, profile in enumerate(profiles):
+                    self.logger.info(f"[{i+1}] {profile['name']}", tag="PROFILES")
+                    
+                # Get profile to use
+                profile_idx = input(f"{Fore.CYAN}Enter the number of the profile to use (or 'c' to cancel): {Style.RESET_ALL}")
+                if profile_idx.lower() == 'c':
+                    continue
+                    
+                try:
+                    idx = int(profile_idx) - 1
+                    if 0 <= idx < len(profiles):
+                        profile_path = profiles[idx]["path"]
+                        url = input(f"{Fore.CYAN}Enter the URL to crawl: {Style.RESET_ALL}")
+                        if url:
+                            # Call the provided crawl callback
+                            await crawl_callback(profile_path, url)
+                        else:
+                            self.logger.error("No URL provided", tag="CRAWL")
+                    else:
+                        self.logger.error("Invalid profile number", tag="PROFILES")
+                except ValueError:
+                    self.logger.error("Please enter a valid number", tag="PROFILES")
+                    
+            elif choice == exit_option:
+                # Exit
+                self.logger.info("Exiting profile management", tag="MENU")
+                break
+                
+            else:
+                self.logger.error(f"Invalid choice. Please enter a number between 1 and {exit_option}.", tag="MENU")

crawl4ai/cache_context.py

@@ -4,7 +4,7 @@ from enum import Enum
 class CacheMode(Enum):
     """
     Defines the caching behavior for web crawling operations.
-    
+
     Modes:
     - ENABLED: Normal caching behavior (read and write)
     - DISABLED: No caching at all
@@ -12,6 +12,7 @@ class CacheMode(Enum):
     - WRITE_ONLY: Only write to cache, don't read
     - BYPASS: Bypass cache for this operation
     """
+
     ENABLED = "enabled"
     DISABLED = "disabled"
     READ_ONLY = "read_only"
@@ -22,32 +23,69 @@ class CacheMode(Enum):
 class CacheContext:
     """
     Encapsulates cache-related decisions and URL handling.
-    
+
     This class centralizes all cache-related logic and URL type checking,
     making the caching behavior more predictable and maintainable.
+
+    Attributes:
+        url (str): The URL being processed.
+        cache_mode (CacheMode): The cache mode for the current operation.
+        always_bypass (bool): If True, bypasses caching for this operation.
+        is_cacheable (bool): True if the URL is cacheable, False otherwise.
+        is_web_url (bool): True if the URL is a web URL, False otherwise.
+        is_local_file (bool): True if the URL is a local file, False otherwise.
+        is_raw_html (bool): True if the URL is raw HTML, False otherwise.
+        _url_display (str): The display name for the URL (web, local file, or raw HTML).
     """
+
     def __init__(self, url: str, cache_mode: CacheMode, always_bypass: bool = False):
+        """
+        Initializes the CacheContext with the provided URL and cache mode.
+
+        Args:
+            url (str): The URL being processed.
+            cache_mode (CacheMode): The cache mode for the current operation.
+            always_bypass (bool): If True, bypasses caching for this operation.
+        """
         self.url = url
         self.cache_mode = cache_mode
         self.always_bypass = always_bypass
-        self.is_cacheable = url.startswith(('http://', 'https://', 'file://'))
-        self.is_web_url = url.startswith(('http://', 'https://'))
+        self.is_cacheable = url.startswith(("http://", "https://", "file://"))
+        self.is_web_url = url.startswith(("http://", "https://"))
         self.is_local_file = url.startswith("file://")
         self.is_raw_html = url.startswith("raw:")
         self._url_display = url if not self.is_raw_html else "Raw HTML"
-    
+
     def should_read(self) -> bool:
-        """Determines if cache should be read based on context."""
+        """
+        Determines if cache should be read based on context.
+
+        How it works:
+        1. If always_bypass is True or is_cacheable is False, return False.
+        2. If cache_mode is ENABLED or READ_ONLY, return True.
+
+        Returns:
+            bool: True if cache should be read, False otherwise.
+        """
         if self.always_bypass or not self.is_cacheable:
             return False
         return self.cache_mode in [CacheMode.ENABLED, CacheMode.READ_ONLY]
-    
+
     def should_write(self) -> bool:
-        """Determines if cache should be written based on context."""
+        """
+        Determines if cache should be written based on context.
+
+        How it works:
+        1. If always_bypass is True or is_cacheable is False, return False.
+        2. If cache_mode is ENABLED or WRITE_ONLY, return True.
+
+        Returns:
+            bool: True if cache should be written, False otherwise.
+        """
         if self.always_bypass or not self.is_cacheable:
             return False
         return self.cache_mode in [CacheMode.ENABLED, CacheMode.WRITE_ONLY]
-    
+
     @property
     def display_url(self) -> str:
         """Returns the URL in display format."""
@@ -58,11 +96,11 @@ def _legacy_to_cache_mode(
     disable_cache: bool = False,
     bypass_cache: bool = False,
     no_cache_read: bool = False,
-    no_cache_write: bool = False
+    no_cache_write: bool = False,
 ) -> CacheMode:
     """
     Converts legacy cache parameters to the new CacheMode enum.
-    
+
     This is an internal function to help transition from the old boolean flags
     to the new CacheMode system.
     """

crawl4ai/chunking_strategy.py

@@ -3,23 +3,52 @@ import re
 from collections import Counter
 import string
 from .model_loader import load_nltk_punkt
-from .utils import *
 
 # Define the abstract base class for chunking strategies
 class ChunkingStrategy(ABC):
-    
+    """
+    Abstract base class for chunking strategies.
+    """
+
     @abstractmethod
     def chunk(self, text: str) -> list:
         """
         Abstract method to chunk the given text.
+
+        Args:
+            text (str): The text to chunk.
+
+        Returns:
+            list: A list of chunks.
         """
         pass
-    
+
+
+# Create an identity chunking strategy f(x) = [x]
+class IdentityChunking(ChunkingStrategy):
+    """
+    Chunking strategy that returns the input text as a single chunk.
+    """
+
+    def chunk(self, text: str) -> list:
+        return [text]
+
+
 # Regex-based chunking
 class RegexChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text based on regular expression patterns.
+    """
+
     def __init__(self, patterns=None, **kwargs):
+        """
+        Initialize the RegexChunking object.
+
+        Args:
+            patterns (list): A list of regular expression patterns to split text.
+        """
         if patterns is None:
-            patterns = [r'\n\n']  # Default split pattern
+            patterns = [r"\n\n"]  # Default split pattern
         self.patterns = patterns
 
     def chunk(self, text: str) -> list:
@@ -30,12 +59,20 @@ class RegexChunking(ChunkingStrategy):
                 new_paragraphs.extend(re.split(pattern, paragraph))
             paragraphs = new_paragraphs
         return paragraphs
-    
-# NLP-based sentence chunking 
+
+
+# NLP-based sentence chunking
 class NlpSentenceChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into sentences using NLTK's sentence tokenizer.
+    """
+
     def __init__(self, **kwargs):
+        """
+        Initialize the NlpSentenceChunking object.
+        """
+        from crawl4ai.le.legacy.model_loader import load_nltk_punkt
         load_nltk_punkt()
-        pass
 
     def chunk(self, text: str) -> list:
         # Improved regex for sentence splitting
@@ -43,18 +80,34 @@ class NlpSentenceChunking(ChunkingStrategy):
         #     r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<![A-Z][A-Z]\.)(?<![A-Za-z]\.)(?<=\.|\?|\!|\n)\s'
         # )
         # sentences = sentence_endings.split(text)
-        # sens =  [sent.strip() for sent in sentences if sent]            
+        # sens =  [sent.strip() for sent in sentences if sent]
         from nltk.tokenize import sent_tokenize
+
         sentences = sent_tokenize(text)
-        sens =  [sent.strip() for sent in sentences]        
-        
+        sens = [sent.strip() for sent in sentences]
+
         return list(set(sens))
-    
+
+
 # Topic-based segmentation using TextTiling
 class TopicSegmentationChunking(ChunkingStrategy):
-    
+    """
+    Chunking strategy that segments text into topics using NLTK's TextTilingTokenizer.
+
+    How it works:
+    1. Segment the text into topics using TextTilingTokenizer
+    2. Extract keywords for each topic segment
+    """
+
     def __init__(self, num_keywords=3, **kwargs):
+        """
+        Initialize the TopicSegmentationChunking object.
+
+        Args:
+            num_keywords (int): The number of keywords to extract for each topic segment.
+        """
         import nltk as nl
+
         self.tokenizer = nl.tokenize.TextTilingTokenizer()
         self.num_keywords = num_keywords
 
@@ -66,8 +119,14 @@ class TopicSegmentationChunking(ChunkingStrategy):
     def extract_keywords(self, text: str) -> list:
         # Tokenize and remove stopwords and punctuation
         import nltk as nl
+
         tokens = nl.toknize.word_tokenize(text)
-        tokens = [token.lower() for token in tokens if token not in nl.corpus.stopwords.words('english') and token not in string.punctuation]
+        tokens = [
+            token.lower()
+            for token in tokens
+            if token not in nl.corpus.stopwords.words("english")
+            and token not in string.punctuation
+        ]
 
         # Calculate frequency distribution
         freq_dist = Counter(tokens)
@@ -78,15 +137,27 @@ class TopicSegmentationChunking(ChunkingStrategy):
         # Segment the text into topics
         segments = self.chunk(text)
         # Extract keywords for each topic segment
-        segments_with_topics = [(segment, self.extract_keywords(segment)) for segment in segments]
+        segments_with_topics = [
+            (segment, self.extract_keywords(segment)) for segment in segments
+        ]
         return segments_with_topics
-    
+
+
 # Fixed-length word chunks
 class FixedLengthWordChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into fixed-length word chunks.
+
+    How it works:
+    1. Split the text into words
+    2. Create chunks of fixed length
+    3. Return the list of chunks
+    """
+
     def __init__(self, chunk_size=100, **kwargs):
         """
         Initialize the fixed-length word chunking strategy with the given chunk size.
-        
+
         Args:
             chunk_size (int): The size of each chunk in words.
         """
@@ -94,15 +165,28 @@ class FixedLengthWordChunking(ChunkingStrategy):
 
     def chunk(self, text: str) -> list:
         words = text.split()
-        return [' '.join(words[i:i + self.chunk_size]) for i in range(0, len(words), self.chunk_size)]
-    
+        return [
+            " ".join(words[i : i + self.chunk_size])
+            for i in range(0, len(words), self.chunk_size)
+        ]
+
+
 # Sliding window chunking
 class SlidingWindowChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into overlapping word chunks.
+
+    How it works:
+    1. Split the text into words
+    2. Create chunks of fixed length
+    3. Return the list of chunks
+    """
+
     def __init__(self, window_size=100, step=50, **kwargs):
         """
         Initialize the sliding window chunking strategy with the given window size and
         step size.
-        
+
         Args:
             window_size (int): The size of the sliding window in words.
             step (int): The step size for sliding the window in words.
@@ -113,27 +197,37 @@ class SlidingWindowChunking(ChunkingStrategy):
     def chunk(self, text: str) -> list:
         words = text.split()
         chunks = []
-        
+
         if len(words) <= self.window_size:
             return [text]
-        
+
         for i in range(0, len(words) - self.window_size + 1, self.step):
-            chunk = ' '.join(words[i:i + self.window_size])
+            chunk = " ".join(words[i : i + self.window_size])
             chunks.append(chunk)
-        
+
         # Handle the last chunk if it doesn't align perfectly
         if i + self.window_size < len(words):
-            chunks.append(' '.join(words[-self.window_size:]))
-        
+            chunks.append(" ".join(words[-self.window_size :]))
+
         return chunks
-    
+
 
 class OverlappingWindowChunking(ChunkingStrategy):
+    """
+    Chunking strategy that splits text into overlapping word chunks.
+
+    How it works:
+    1. Split the text into words using whitespace
+    2. Create chunks of fixed length equal to the window size
+    3. Slide the window by the overlap size
+    4. Return the list of chunks
+    """
+
     def __init__(self, window_size=1000, overlap=100, **kwargs):
         """
         Initialize the overlapping window chunking strategy with the given window size and
         overlap size.
-        
+
         Args:
             window_size (int): The size of the window in words.
             overlap (int): The size of the overlap between consecutive chunks in words.
@@ -144,19 +238,19 @@ class OverlappingWindowChunking(ChunkingStrategy):
     def chunk(self, text: str) -> list:
         words = text.split()
         chunks = []
-        
+
         if len(words) <= self.window_size:
             return [text]
-        
+
         start = 0
         while start < len(words):
             end = start + self.window_size
-            chunk = ' '.join(words[start:end])
+            chunk = " ".join(words[start:end])
             chunks.append(chunk)
-            
+
             if end >= len(words):
                 break
-            
+
             start = end - self.overlap
-        
-        return chunks
+
+        return chunks

crawl4ai/cli.py

@@ -0,0 +1,782 @@
+import click
+import os
+import time
+import datetime
+import sys
+import shutil
+import humanize
+from typing import Dict, Any, Optional, List
+import json
+import yaml
+import anyio
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.prompt import Prompt, Confirm
+from rich.style import Style
+
+from crawl4ai import (
+    CacheMode,
+    AsyncWebCrawler, 
+    CrawlResult,
+    BrowserConfig, 
+    CrawlerRunConfig,
+    LLMExtractionStrategy, 
+    JsonCssExtractionStrategy,
+    JsonXPathExtractionStrategy,
+    BM25ContentFilter, 
+    PruningContentFilter,
+    BrowserProfiler
+)
+from litellm import completion
+from pathlib import Path
+
+from crawl4ai.async_configs import LlmConfig
+
+# Initialize rich console
+console = Console()
+
+def get_global_config() -> dict:
+    config_dir = Path.home() / ".crawl4ai"
+    config_file = config_dir / "global.yml"
+    
+    if not config_file.exists():
+        config_dir.mkdir(parents=True, exist_ok=True)
+        return {}
+        
+    with open(config_file) as f:
+        return yaml.safe_load(f) or {}
+
+def save_global_config(config: dict):
+    config_file = Path.home() / ".crawl4ai" / "global.yml"
+    with open(config_file, "w") as f:
+        yaml.dump(config, f)
+
+def setup_llm_config() -> tuple[str, str]:
+    config = get_global_config()
+    provider = config.get("DEFAULT_LLM_PROVIDER")
+    token = config.get("DEFAULT_LLM_PROVIDER_TOKEN")
+    
+    if not provider:
+        click.echo("\nNo default LLM provider configured.")
+        click.echo("Provider format: 'company/model' (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')")
+        click.echo("See available providers at: https://docs.litellm.ai/docs/providers")
+        provider = click.prompt("Enter provider")
+        
+    if not provider.startswith("ollama/"):
+        if not token:
+            token = click.prompt("Enter API token for " + provider, hide_input=True)
+    else:
+        token = "no-token"
+    
+    if not config.get("DEFAULT_LLM_PROVIDER") or not config.get("DEFAULT_LLM_PROVIDER_TOKEN"):
+        config["DEFAULT_LLM_PROVIDER"] = provider
+        config["DEFAULT_LLM_PROVIDER_TOKEN"] = token
+        save_global_config(config)
+        click.echo("\nConfiguration saved to ~/.crawl4ai/global.yml")
+    
+    return provider, token
+
+async def stream_llm_response(url: str, markdown: str, query: str, provider: str, token: str):
+    response = completion(
+        model=provider,
+        api_key=token,
+        messages=[
+            {
+                "content": f"You are Crawl4ai assistant, answering user question based on the provided context which is crawled from {url}.",
+                "role": "system"
+            },
+            {
+                "content": f"<|start of context|>\n{markdown}\n<|end of context|>\n\n{query}",
+                "role": "user"
+            },
+        ],
+        stream=True,
+    )
+    
+    for chunk in response:
+        if content := chunk["choices"][0]["delta"].get("content"):
+            print(content, end="", flush=True)
+    print()  # New line at end
+
+
+
+def parse_key_values(ctx, param, value) -> Dict[str, Any]:
+    if not value:
+        return {}
+    result = {}
+    pairs = value.split(',')
+    for pair in pairs:
+        try:
+            k, v = pair.split('=', 1)
+            # Handle common value types 
+            if v.lower() == 'true': v = True
+            elif v.lower() == 'false': v = False
+            elif v.isdigit(): v = int(v)
+            elif v.replace('.','',1).isdigit(): v = float(v)
+            elif v.startswith('[') and v.endswith(']'):
+                v = [x.strip() for x in v[1:-1].split(',') if x.strip()]
+            elif v.startswith('{') and v.endswith('}'):
+                try:
+                    v = json.loads(v)
+                except json.JSONDecodeError:
+                    raise click.BadParameter(f'Invalid JSON object: {v}')
+            result[k.strip()] = v
+        except ValueError:
+            raise click.BadParameter(f'Invalid key=value pair: {pair}')
+    return result
+
+def load_config_file(path: Optional[str]) -> dict:
+    if not path:
+        return {}
+    
+    try:
+        with open(path) as f:
+            if path.endswith((".yaml", ".yml")):
+                return yaml.safe_load(f)
+            return json.load(f)
+    except Exception as e:
+        raise click.BadParameter(f'Error loading config file {path}: {str(e)}')
+
+def load_schema_file(path: Optional[str]) -> dict:
+    if not path:
+        return None
+    return load_config_file(path)
+
+async def run_crawler(url: str, browser_cfg: BrowserConfig, crawler_cfg: CrawlerRunConfig, verbose: bool):
+    if verbose:
+        click.echo("Starting crawler with configurations:")
+        click.echo(f"Browser config: {browser_cfg.dump()}")
+        click.echo(f"Crawler config: {crawler_cfg.dump()}")
+
+    async with AsyncWebCrawler(config=browser_cfg) as crawler:
+        try:
+            result = await crawler.arun(url=url, config=crawler_cfg)
+            return result
+        except Exception as e:
+            raise click.ClickException(f"Crawling failed: {str(e)}")
+
+def show_examples():
+    examples = """
+🚀 Crawl4AI CLI Examples
+
+1️⃣  Basic Usage:
+    # Simple crawl with default settings
+    crwl https://example.com
+
+    # Get markdown output
+    crwl https://example.com -o markdown
+
+    # Verbose JSON output with cache bypass
+    crwl https://example.com -o json -v --bypass-cache
+
+2️⃣  Using Config Files:
+    # Using browser and crawler configs
+    crwl https://example.com -B browser.yml -C crawler.yml
+
+    # CSS-based extraction
+    crwl https://example.com -e extract_css.yml -s css_schema.json -o json
+
+    # LLM-based extraction
+    crwl https://example.com -e extract_llm.yml -s llm_schema.json -o json
+
+3️⃣  Direct Parameters:
+    # Browser settings
+    crwl https://example.com -b "headless=true,viewport_width=1280,user_agent_mode=random"
+
+    # Crawler settings
+    crwl https://example.com -c "css_selector=#main,delay_before_return_html=2,scan_full_page=true"
+
+4️⃣  Profile Management for Identity-Based Crawling:
+    # Launch interactive profile manager
+    crwl profiles
+
+    # Create, list, and delete browser profiles for identity-based crawling
+    # Use a profile for crawling (keeps you logged in)
+    crwl https://example.com -p my-profile-name
+
+    # Example: Crawl a site that requires login
+    # 1. First create a profile and log in:
+    crwl profiles
+    # 2. Then use that profile to crawl the authenticated site:
+    crwl https://site-requiring-login.com/dashboard -p my-profile-name
+
+5️⃣  Sample Config Files:
+
+browser.yml:
+    headless: true
+    viewport_width: 1280
+    user_agent_mode: "random"
+    verbose: true
+    ignore_https_errors: true
+
+extract_css.yml:
+    type: "json-css"
+    params:
+        verbose: true
+
+css_schema.json:
+    {
+      "name": "ArticleExtractor",
+      "baseSelector": ".article",
+      "fields": [
+        {
+          "name": "title",
+          "selector": "h1.title",
+          "type": "text"
+        },
+        {
+          "name": "link",
+          "selector": "a.read-more",
+          "type": "attribute",
+          "attribute": "href"
+        }
+      ]
+    }
+
+extract_llm.yml:
+    type: "llm"
+    provider: "openai/gpt-4"
+    instruction: "Extract all articles with their titles and links"
+    api_token: "your-token"
+    params:
+        temperature: 0.3
+        max_tokens: 1000
+
+llm_schema.json:
+    {
+      "title": "Article",
+      "type": "object",
+      "properties": {
+        "title": {
+          "type": "string",
+          "description": "The title of the article"
+        },
+        "link": {
+          "type": "string",
+          "description": "URL to the full article"
+        }
+      }
+    }
+
+6️⃣  Advanced Usage:
+    # Combine configs with direct parameters
+    crwl https://example.com -B browser.yml -b "headless=false,viewport_width=1920"
+
+    # Full extraction pipeline
+    crwl https://example.com \\
+        -B browser.yml \\
+        -C crawler.yml \\
+        -e extract_llm.yml \\
+        -s llm_schema.json \\
+        -o json \\
+        -v
+
+    # Content filtering with BM25
+    crwl https://example.com \\
+        -f filter_bm25.yml \\
+        -o markdown-fit
+
+    # Authenticated crawling with profile
+    crwl https://login-required-site.com \\
+        -p my-authenticated-profile \\
+        -c "css_selector=.dashboard-content" \\
+        -o markdown
+
+For more documentation visit: https://github.com/unclecode/crawl4ai
+
+7️⃣  Q&A with LLM:
+    # Ask a question about the content
+    crwl https://example.com -q "What is the main topic discussed?"
+
+    # First view content, then ask questions
+    crwl https://example.com -o markdown  # See the crawled content first
+    crwl https://example.com -q "Summarize the key points"
+    crwl https://example.com -q "What are the conclusions?"
+
+    # Advanced crawling with Q&A
+    crwl https://example.com \\
+        -B browser.yml \\
+        -c "css_selector=article,scan_full_page=true" \\
+        -q "What are the pros and cons mentioned?"
+
+    Note: First time using -q will prompt for LLM provider and API token.
+    These will be saved in ~/.crawl4ai/global.yml for future use.
+    
+    Supported provider format: 'company/model'
+    Examples:
+      - ollama/llama3.3
+      - openai/gpt-4
+      - anthropic/claude-3-sonnet
+      - cohere/command
+      - google/gemini-pro
+    
+    See full list of providers: https://docs.litellm.ai/docs/providers
+
+8️⃣ Profile Management:
+    # Launch interactive profile manager
+    crwl profiles
+
+    # Create a profile and use it for crawling
+    crwl profiles  # Create and set up your profile interactively
+    crwl https://example.com -p my-profile-name  # Use profile for crawling
+
+    # Example workflow for authenticated site
+    # 1. First create a profile and log in to the site:
+    crwl profiles  # Select "Create new profile" option
+    # 2. Then use that profile to crawl authenticated content:
+    crwl https://site-requiring-login.com/dashboard -p my-profile-name
+"""
+    click.echo(examples)
+
+def get_directory_size(path: str) -> int:
+    """Calculate the total size of a directory in bytes"""
+    total_size = 0
+    for dirpath, _, filenames in os.walk(path):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            if not os.path.islink(fp):
+                total_size += os.path.getsize(fp)
+    return total_size
+
+def display_profiles_table(profiles: List[Dict[str, Any]]):
+    """Display a rich table of browser profiles"""
+    if not profiles:
+        console.print(Panel("[yellow]No profiles found. Create one with the 'create' command.[/yellow]", 
+                          title="Browser Profiles", border_style="blue"))
+        return
+    
+    table = Table(title="Browser Profiles", show_header=True, header_style="bold cyan", border_style="blue")
+    table.add_column("#", style="dim", width=4)
+    table.add_column("Name", style="cyan", no_wrap=True)
+    table.add_column("Path", style="green")
+    table.add_column("Created", style="yellow")
+    table.add_column("Browser", style="magenta")
+    table.add_column("Size", style="blue", justify="right")
+    
+    for i, profile in enumerate(profiles):
+        # Calculate folder size
+        size = get_directory_size(profile["path"])
+        human_size = humanize.naturalsize(size)
+        
+        # Format creation date
+        created = profile["created"].strftime("%Y-%m-%d %H:%M")
+        
+        # Add row to table
+        table.add_row(
+            str(i+1), 
+            profile["name"], 
+            profile["path"], 
+            created, 
+            profile["type"].capitalize(), 
+            human_size
+        )
+    
+    console.print(table)
+
+async def create_profile_interactive(profiler: BrowserProfiler):
+    """Interactive profile creation wizard"""
+    console.print(Panel("[bold cyan]Create Browser Profile[/bold cyan]\n"
+                      "This will open a browser window for you to set up your identity.\n"
+                      "Log in to sites, adjust settings, then press 'q' to save.",
+                      border_style="cyan"))
+    
+    profile_name = Prompt.ask("[cyan]Enter profile name[/cyan]", default=f"profile_{int(time.time())}")
+    
+    console.print("[cyan]Creating profile...[/cyan]")
+    console.print("[yellow]A browser window will open. After logging in to sites, press 'q' in this terminal to save.[/yellow]")
+    
+    # Create the profile
+    try:
+        profile_path = await profiler.create_profile(profile_name)
+        
+        if profile_path:
+            console.print(f"[green]Profile successfully created at:[/green] {profile_path}")
+        else:
+            console.print("[red]Failed to create profile.[/red]")
+    except Exception as e:
+        console.print(f"[red]Error creating profile: {str(e)}[/red]")
+
+def delete_profile_interactive(profiler: BrowserProfiler):
+    """Interactive profile deletion"""
+    profiles = profiler.list_profiles()
+    
+    if not profiles:
+        console.print("[yellow]No profiles found to delete.[/yellow]")
+        return
+    
+    # Display profiles
+    display_profiles_table(profiles)
+    
+    # Get profile selection
+    idx = Prompt.ask(
+        "[red]Enter number of profile to delete[/red]", 
+        console=console,
+        choices=[str(i+1) for i in range(len(profiles))],
+        show_choices=False
+    )
+    
+    try:
+        idx = int(idx) - 1
+        profile = profiles[idx]
+        
+        # Confirm deletion
+        if Confirm.ask(f"[red]Are you sure you want to delete profile '{profile['name']}'?[/red]"):
+            success = profiler.delete_profile(profile["path"])
+            
+            if success:
+                console.print(f"[green]Profile '{profile['name']}' deleted successfully.[/green]")
+            else:
+                console.print(f"[red]Failed to delete profile '{profile['name']}'.[/red]")
+    except (ValueError, IndexError):
+        console.print("[red]Invalid selection.[/red]")
+        
+async def crawl_with_profile_cli(profile_path, url):
+    """Use a profile to crawl a website via CLI"""
+    console.print(f"[cyan]Crawling [bold]{url}[/bold] using profile at [bold]{profile_path}[/bold][/cyan]")
+    
+    # Create browser config with the profile
+    browser_cfg = BrowserConfig(
+        headless=False,  # Set to False to see the browser in action
+        use_managed_browser=True,
+        user_data_dir=profile_path
+    )
+    
+    # Default crawler config
+    crawler_cfg = CrawlerRunConfig()
+    
+    # Ask for output format
+    output_format = Prompt.ask(
+        "[cyan]Output format[/cyan]",
+        choices=["all", "json", "markdown", "md", "title"],
+        default="markdown"
+    )
+    
+    try:
+        # Run the crawler
+        result = await run_crawler(url, browser_cfg, crawler_cfg, True)
+        
+        # Handle output
+        if output_format == "all":
+            console.print(json.dumps(result.model_dump(), indent=2))
+        elif output_format == "json":
+            console.print(json.dumps(json.loads(result.extracted_content), indent=2))
+        elif output_format in ["markdown", "md"]:
+            console.print(result.markdown.raw_markdown)
+        elif output_format == "title":
+            console.print(result.metadata.get("title", "No title found"))
+        
+        console.print(f"[green]Successfully crawled[/green] {url}")
+        return result
+    except Exception as e:
+        console.print(f"[red]Error crawling:[/red] {str(e)}")
+        return None
+        
+async def use_profile_to_crawl():
+    """Interactive profile selection for crawling"""
+    profiler = BrowserProfiler()
+    profiles = profiler.list_profiles()
+    
+    if not profiles:
+        console.print("[yellow]No profiles found. Create one first.[/yellow]")
+        return
+        
+    # Display profiles
+    display_profiles_table(profiles)
+    
+    # Get profile selection
+    idx = Prompt.ask(
+        "[cyan]Enter number of profile to use[/cyan]", 
+        console=console,
+        choices=[str(i+1) for i in range(len(profiles))],
+        show_choices=False
+    )
+    
+    try:
+        idx = int(idx) - 1
+        profile = profiles[idx]
+        
+        # Get URL
+        url = Prompt.ask("[cyan]Enter URL to crawl[/cyan]")
+        if url:
+            # Crawl with the selected profile
+            await crawl_with_profile_cli(profile["path"], url)
+        else:
+            console.print("[red]No URL provided[/red]")
+    except (ValueError, IndexError):
+        console.print("[red]Invalid selection[/red]")
+
+async def manage_profiles():
+    """Interactive profile management menu"""
+    profiler = BrowserProfiler()
+    
+    options = {
+        "1": "List profiles",
+        "2": "Create new profile",
+        "3": "Delete profile",
+        "4": "Use a profile to crawl a website",
+        "5": "Exit",
+    }
+    
+    while True:
+        console.print(Panel("[bold cyan]Browser Profile Manager[/bold cyan]", border_style="cyan"))
+        
+        for key, value in options.items():
+            color = "green" if key == "1" else "yellow" if key == "2" else "red" if key == "3" else "blue" if key == "4" else "cyan"
+            console.print(f"[{color}]{key}[/{color}]. {value}")
+        
+        choice = Prompt.ask("Enter choice", choices=list(options.keys()), default="1")
+        
+        if choice == "1":
+            # List profiles
+            profiles = profiler.list_profiles()
+            display_profiles_table(profiles)
+        
+        elif choice == "2":
+            # Create profile
+            await create_profile_interactive(profiler)
+        
+        elif choice == "3":
+            # Delete profile
+            delete_profile_interactive(profiler)
+            
+        elif choice == "4":
+            # Use profile to crawl
+            await use_profile_to_crawl()
+        
+        elif choice == "5":
+            # Exit
+            console.print("[cyan]Exiting profile manager.[/cyan]")
+            break
+        
+        # Add a separator between operations
+        console.print("\n")
+
+@click.group(context_settings={"help_option_names": ["-h", "--help"]})
+def cli():
+    """Crawl4AI CLI - Web content extraction and browser profile management tool"""
+    pass
+
+@cli.command("crawl")
+@click.argument("url", required=True)
+@click.option("--browser-config", "-B", type=click.Path(exists=True), help="Browser config file (YAML/JSON)")
+@click.option("--crawler-config", "-C", type=click.Path(exists=True), help="Crawler config file (YAML/JSON)")
+@click.option("--filter-config", "-f", type=click.Path(exists=True), help="Content filter config file")
+@click.option("--extraction-config", "-e", type=click.Path(exists=True), help="Extraction strategy config file")
+@click.option("--schema", "-s", type=click.Path(exists=True), help="JSON schema for extraction")
+@click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2")
+@click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2")
+@click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all")
+@click.option("--bypass-cache", is_flag=True, default=True, help="Bypass cache when crawling")
+@click.option("--question", "-q", help="Ask a question about the crawled content")
+@click.option("--verbose", "-v", is_flag=True)
+@click.option("--profile", "-p", help="Use a specific browser profile (by name)")
+def crawl_cmd(url: str, browser_config: str, crawler_config: str, filter_config: str, 
+           extraction_config: str, schema: str, browser: Dict, crawler: Dict,
+           output: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
+    """Crawl a website and extract content
+    
+    Simple Usage:
+        crwl crawl https://example.com
+    """
+    
+    # Handle profile option
+    if profile:
+        profiler = BrowserProfiler()
+        profile_path = profiler.get_profile_path(profile)
+        
+        if not profile_path:
+            profiles = profiler.list_profiles()
+            
+            if profiles:
+                console.print(f"[red]Profile '{profile}' not found. Available profiles:[/red]")
+                display_profiles_table(profiles)
+            else:
+                console.print("[red]No profiles found. Create one with 'crwl profiles'[/red]")
+            
+            return
+        
+        # Include the profile in browser config
+        if not browser:
+            browser = {}
+        browser["user_data_dir"] = profile_path
+        browser["use_managed_browser"] = True
+        
+        if verbose:
+            console.print(f"[green]Using browser profile:[/green] {profile}")
+            
+    try:
+        # Load base configurations
+        browser_cfg = BrowserConfig.load(load_config_file(browser_config))
+        crawler_cfg = CrawlerRunConfig.load(load_config_file(crawler_config))
+        
+        # Override with CLI params
+        if browser:
+            browser_cfg = browser_cfg.clone(**browser)
+        if crawler:
+            crawler_cfg = crawler_cfg.clone(**crawler)
+            
+        # Handle content filter config
+        if filter_config:
+            filter_conf = load_config_file(filter_config)
+            if filter_conf["type"] == "bm25":
+                crawler_cfg.content_filter = BM25ContentFilter(
+                    user_query=filter_conf.get("query"),
+                    bm25_threshold=filter_conf.get("threshold", 1.0)
+                )
+            elif filter_conf["type"] == "pruning":
+                crawler_cfg.content_filter = PruningContentFilter(
+                    user_query=filter_conf.get("query"),
+                    threshold=filter_conf.get("threshold", 0.48)
+                )
+                
+        # Handle extraction strategy
+        if extraction_config:
+            extract_conf = load_config_file(extraction_config)
+            schema_data = load_schema_file(schema)
+            
+            # Check if type does not exist show proper message
+            if not extract_conf.get("type"):
+                raise click.ClickException("Extraction type not specified")
+            if extract_conf["type"] not in ["llm", "json-css", "json-xpath"]:
+                raise click.ClickException(f"Invalid extraction type: {extract_conf['type']}")
+            
+            if extract_conf["type"] == "llm":
+                # if no provider show error emssage
+                if not extract_conf.get("provider") or not extract_conf.get("api_token"):
+                    raise click.ClickException("LLM provider and API token are required for LLM extraction")
+
+                crawler_cfg.extraction_strategy = LLMExtractionStrategy(
+                    llmConfig=LlmConfig(provider=extract_conf["provider"], api_token=extract_conf["api_token"]),
+                    instruction=extract_conf["instruction"],
+                    schema=schema_data,
+                    **extract_conf.get("params", {})
+                )
+            elif extract_conf["type"] == "json-css":
+                crawler_cfg.extraction_strategy = JsonCssExtractionStrategy(
+                    schema=schema_data
+                )
+            elif extract_conf["type"] == "json-xpath":
+                crawler_cfg.extraction_strategy = JsonXPathExtractionStrategy(
+                    schema=schema_data
+                )
+                
+
+        # No cache
+        if bypass_cache:
+            crawler_cfg.cache_mode = CacheMode.BYPASS
+        
+        # Run crawler
+        result : CrawlResult = anyio.run(
+            run_crawler,
+            url,
+            browser_cfg,
+            crawler_cfg,
+            verbose
+        )
+
+        # Handle question
+        if question:
+            provider, token = setup_llm_config()
+            markdown = result.markdown.raw_markdown
+            anyio.run(stream_llm_response, url, markdown, question, provider, token)
+            return
+        
+        # Handle output
+        if output == "all":
+            click.echo(json.dumps(result.model_dump(), indent=2))
+        elif output == "json":
+            click.echo(json.dumps(json.loads(result.extracted_content), indent=2))
+        elif output in ["markdown", "md"]:
+            click.echo(result.markdown.raw_markdown)
+        elif output in ["markdown-fit", "md-fit"]:
+            click.echo(result.markdown.fit_markdown)
+            
+    except Exception as e:
+        raise click.ClickException(str(e))
+
+@cli.command("examples")
+def examples_cmd():
+    """Show usage examples"""
+    show_examples()
+
+@cli.command("profiles")
+def profiles_cmd():
+    """Manage browser profiles interactively
+    
+    Launch an interactive browser profile manager where you can:
+    - List all existing profiles
+    - Create new profiles for authenticated browsing
+    - Delete unused profiles
+    """
+    # Run interactive profile manager
+    anyio.run(manage_profiles)
+
+@cli.command(name="")
+@click.argument("url", required=False)
+@click.option("--example", is_flag=True, help="Show usage examples")
+@click.option("--browser-config", "-B", type=click.Path(exists=True), help="Browser config file (YAML/JSON)")
+@click.option("--crawler-config", "-C", type=click.Path(exists=True), help="Crawler config file (YAML/JSON)")
+@click.option("--filter-config", "-f", type=click.Path(exists=True), help="Content filter config file")
+@click.option("--extraction-config", "-e", type=click.Path(exists=True), help="Extraction strategy config file")
+@click.option("--schema", "-s", type=click.Path(exists=True), help="JSON schema for extraction")
+@click.option("--browser", "-b", type=str, callback=parse_key_values, help="Browser parameters as key1=value1,key2=value2")
+@click.option("--crawler", "-c", type=str, callback=parse_key_values, help="Crawler parameters as key1=value1,key2=value2")
+@click.option("--output", "-o", type=click.Choice(["all", "json", "markdown", "md", "markdown-fit", "md-fit"]), default="all")
+@click.option("--bypass-cache", is_flag=True, default=True, help="Bypass cache when crawling")
+@click.option("--question", "-q", help="Ask a question about the crawled content")
+@click.option("--verbose", "-v", is_flag=True)
+@click.option("--profile", "-p", help="Use a specific browser profile (by name)")
+def default(url: str, example: bool, browser_config: str, crawler_config: str, filter_config: str, 
+        extraction_config: str, schema: str, browser: Dict, crawler: Dict,
+        output: str, bypass_cache: bool, question: str, verbose: bool, profile: str):
+    """Crawl4AI CLI - Web content extraction tool
+
+    Simple Usage:
+        crwl https://example.com
+    
+    Run with --example to see detailed usage examples.
+    
+    Other commands:
+        crwl profiles   - Manage browser profiles for identity-based crawling
+        crwl crawl      - Crawl a website with advanced options
+        crwl examples   - Show more usage examples
+    """
+
+    if example:
+        show_examples()
+        return
+        
+    if not url:
+        # Show help without error message
+        ctx = click.get_current_context()
+        click.echo(ctx.get_help())
+        return
+        
+    # Forward to crawl command
+    ctx = click.get_current_context()
+    ctx.invoke(
+        crawl_cmd, 
+        url=url, 
+        browser_config=browser_config,
+        crawler_config=crawler_config,
+        filter_config=filter_config,
+        extraction_config=extraction_config, 
+        schema=schema,
+        browser=browser,
+        crawler=crawler,
+        output=output,
+        bypass_cache=bypass_cache,
+        question=question,
+        verbose=verbose,
+        profile=profile
+    )
+
+def main():
+    import sys
+    if len(sys.argv) < 2 or sys.argv[1] not in cli.commands:
+        sys.argv.insert(1, "crawl")
+    cli()
+
+if __name__ == "__main__":
+    main()

crawl4ai/config.py

@@ -8,39 +8,71 @@ DEFAULT_PROVIDER = "openai/gpt-4o-mini"
 MODEL_REPO_BRANCH = "new-release-0.0.2"
 # Provider-model dictionary, ONLY used when the extraction strategy is LLMExtractionStrategy
 PROVIDER_MODELS = {
-    "ollama/llama3": "no-token-needed", # Any model from Ollama no need for API token
+    "ollama/llama3": "no-token-needed",  # Any model from Ollama no need for API token
     "groq/llama3-70b-8192": os.getenv("GROQ_API_KEY"),
     "groq/llama3-8b-8192": os.getenv("GROQ_API_KEY"),
     "openai/gpt-4o-mini": os.getenv("OPENAI_API_KEY"),
     "openai/gpt-4o": os.getenv("OPENAI_API_KEY"),
+    "openai/o1-mini": os.getenv("OPENAI_API_KEY"),
+    "openai/o1-preview": os.getenv("OPENAI_API_KEY"),
+    "openai/o3-mini": os.getenv("OPENAI_API_KEY"),
+    "openai/o3-mini-high": os.getenv("OPENAI_API_KEY"),
     "anthropic/claude-3-haiku-20240307": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-opus-20240229": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-sonnet-20240229": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-5-sonnet-20240620": os.getenv("ANTHROPIC_API_KEY"),
+    "gemini/gemini-pro": os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-1.5-pro': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash-exp': os.getenv("GEMINI_API_KEY"),
+    'gemini/gemini-2.0-flash-lite-preview-02-05': os.getenv("GEMINI_API_KEY"),
+    "deepseek/deepseek-chat": os.getenv("DEEPSEEK_API_KEY"),
 }
 
 # Chunk token threshold
-CHUNK_TOKEN_THRESHOLD = 2 ** 11 # 2048 tokens
+CHUNK_TOKEN_THRESHOLD = 2**11  # 2048 tokens
 OVERLAP_RATE = 0.1
 WORD_TOKEN_RATE = 1.3
 
-# Threshold for the minimum number of word in a HTML tag to be considered 
+# Threshold for the minimum number of word in a HTML tag to be considered
 MIN_WORD_THRESHOLD = 1
 IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD = 1
 
-IMPORTANT_ATTRS = ['src', 'href', 'alt', 'title', 'width', 'height'] 
-ONLY_TEXT_ELIGIBLE_TAGS = ['b', 'i', 'u', 'span', 'del', 'ins', 'sub', 'sup', 'strong', 'em', 'code', 'kbd', 'var', 's', 'q', 'abbr', 'cite', 'dfn', 'time', 'small', 'mark']
+IMPORTANT_ATTRS = ["src", "href", "alt", "title", "width", "height"]
+ONLY_TEXT_ELIGIBLE_TAGS = [
+    "b",
+    "i",
+    "u",
+    "span",
+    "del",
+    "ins",
+    "sub",
+    "sup",
+    "strong",
+    "em",
+    "code",
+    "kbd",
+    "var",
+    "s",
+    "q",
+    "abbr",
+    "cite",
+    "dfn",
+    "time",
+    "small",
+    "mark",
+]
 SOCIAL_MEDIA_DOMAINS = [
-                            'facebook.com',
-                            'twitter.com',
-                            'x.com',
-                            'linkedin.com',
-                            'instagram.com',
-                            'pinterest.com',
-                            'tiktok.com',
-                            'snapchat.com',
-                            'reddit.com',
-                        ]
+    "facebook.com",
+    "twitter.com",
+    "x.com",
+    "linkedin.com",
+    "instagram.com",
+    "pinterest.com",
+    "tiktok.com",
+    "snapchat.com",
+    "reddit.com",
+]
 
 # Threshold for the Image extraction - Range is 1 to 6
 # Images are scored based on point based system, to filter based on usefulness. Points are assigned
@@ -56,4 +88,7 @@ MAX_METRICS_HISTORY = 1000
 
 NEED_MIGRATION = True
 URL_LOG_SHORTEN_LENGTH = 30
-SHOW_DEPRECATION_WARNINGS = True
+SHOW_DEPRECATION_WARNINGS = True
+SCREENSHOT_HEIGHT_TRESHOLD = 10000
+PAGE_TIMEOUT = 60000
+DOWNLOAD_PAGE_TIMEOUT = 60000

crawl4ai/configs/__init__.py

@@ -0,0 +1,2 @@
+from .proxy_config import ProxyConfig
+__all__ = ["ProxyConfig"]

crawl4ai/configs/proxy_config.py

@@ -0,0 +1,113 @@
+import os
+from typing import Dict, List, Optional
+
+
+class ProxyConfig:
+    def __init__(
+        self,
+        server: str,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        ip: Optional[str] = None,
+    ):
+        """Configuration class for a single proxy.
+        
+        Args:
+            server: Proxy server URL (e.g., "http://127.0.0.1:8080")
+            username: Optional username for proxy authentication
+            password: Optional password for proxy authentication
+            ip: Optional IP address for verification purposes
+        """
+        self.server = server
+        self.username = username
+        self.password = password
+        
+        # Extract IP from server if not explicitly provided
+        self.ip = ip or self._extract_ip_from_server()
+    
+    def _extract_ip_from_server(self) -> Optional[str]:
+        """Extract IP address from server URL."""
+        try:
+            # Simple extraction assuming http://ip:port format
+            if "://" in self.server:
+                parts = self.server.split("://")[1].split(":")
+                return parts[0]
+            else:
+                parts = self.server.split(":")
+                return parts[0]
+        except Exception:
+            return None
+    
+    @staticmethod
+    def from_string(proxy_str: str) -> "ProxyConfig":
+        """Create a ProxyConfig from a string in the format 'ip:port:username:password'."""
+        parts = proxy_str.split(":")
+        if len(parts) == 4:  # ip:port:username:password
+            ip, port, username, password = parts
+            return ProxyConfig(
+                server=f"http://{ip}:{port}",
+                username=username,
+                password=password,
+                ip=ip
+            )
+        elif len(parts) == 2:  # ip:port only
+            ip, port = parts
+            return ProxyConfig(
+                server=f"http://{ip}:{port}",
+                ip=ip
+            )
+        else:
+            raise ValueError(f"Invalid proxy string format: {proxy_str}")
+    
+    @staticmethod
+    def from_dict(proxy_dict: Dict) -> "ProxyConfig":
+        """Create a ProxyConfig from a dictionary."""
+        return ProxyConfig(
+            server=proxy_dict.get("server"),
+            username=proxy_dict.get("username"),
+            password=proxy_dict.get("password"),
+            ip=proxy_dict.get("ip")
+        )
+    
+    @staticmethod
+    def from_env(env_var: str = "PROXIES") -> List["ProxyConfig"]:
+        """Load proxies from environment variable.
+        
+        Args:
+            env_var: Name of environment variable containing comma-separated proxy strings
+            
+        Returns:
+            List of ProxyConfig objects
+        """
+        proxies = []
+        try:
+            proxy_list = os.getenv(env_var, "").split(",")
+            for proxy in proxy_list:
+                if not proxy:
+                    continue
+                proxies.append(ProxyConfig.from_string(proxy))
+        except Exception as e:
+            print(f"Error loading proxies from environment: {e}")
+        return proxies
+    
+    def to_dict(self) -> Dict:
+        """Convert to dictionary representation."""
+        return {
+            "server": self.server,
+            "username": self.username,
+            "password": self.password,
+            "ip": self.ip
+        }
+    
+    def clone(self, **kwargs) -> "ProxyConfig":
+        """Create a copy of this configuration with updated values.
+
+        Args:
+            **kwargs: Key-value pairs of configuration options to update
+
+        Returns:
+            ProxyConfig: A new instance with the specified updates
+        """
+        config_dict = self.to_dict()
+        config_dict.update(kwargs)
+        return ProxyConfig.from_dict(config_dict)

crawl4ai/crawlers/amazon_product/crawler.py

@@ -0,0 +1,20 @@
+from crawl4ai.hub import BaseCrawler
+
+__meta__ = {
+    "version": "1.2.0",
+    "tested_on": ["amazon.com"],
+    "rate_limit": "50 RPM",
+    "schema": {"product": ["name", "price"]}
+}
+
+class AmazonProductCrawler(BaseCrawler):
+    async def run(self, url: str, **kwargs) -> str:
+        try:
+            self.logger.info(f"Crawling {url}")
+            return '{"product": {"name": "Test Amazon Product"}}'
+        except Exception as e:
+            self.logger.error(f"Crawl failed: {str(e)}")
+            return json.dumps({
+                "error": str(e),
+                "metadata": self.meta  # Include meta in error response
+            })            

crawl4ai/crawlers/google_search/crawler.py

@@ -0,0 +1,130 @@
+from crawl4ai import BrowserConfig, AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai.hub import BaseCrawler
+from crawl4ai.utils import optimize_html, get_home_folder
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from pathlib import Path
+import json
+import os
+from typing import Dict
+
+
+class GoogleSearchCrawler(BaseCrawler):
+    __meta__ = {
+        "version": "1.0.0",
+        "tested_on": ["google.com/search*"],
+        "rate_limit": "10 RPM",
+        "description": "Crawls Google Search results (text + images)",
+    }
+
+    def __init__(self):
+        super().__init__()
+        self.js_script = (Path(__file__).parent /
+                          "script.js").read_text()
+
+    async def run(self, url="", query: str = "", search_type: str = "text", schema_cache_path = None, **kwargs) -> str:
+        """Crawl Google Search results for a query"""
+        url = f"https://www.google.com/search?q={query}&gl=sg&hl=en" if search_type == "text" else f"https://www.google.com/search?q={query}&gl=sg&hl=en&tbs=qdr:d&udm=2"
+        if kwargs.get("page_start", 1) > 1:
+            url = f"{url}&start={kwargs['page_start'] * 10}"
+        if kwargs.get("page_length", 1) > 1:
+            url = f"{url}&num={kwargs['page_length']}"
+            
+        browser_config = BrowserConfig(headless=True, verbose=True)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            config = CrawlerRunConfig(
+                cache_mode=kwargs.get("cache_mode", CacheMode.BYPASS),
+                keep_attrs=["id", "class"],
+                keep_data_attributes=True,
+                delay_before_return_html=kwargs.get(
+                    "delay", 2 if search_type == "image" else 1),
+                js_code=self.js_script if search_type == "image" else None,
+            )
+
+            result = await crawler.arun(url=url, config=config)
+            if not result.success:
+                return json.dumps({"error": result.error})
+
+            if search_type == "image":
+                if result.js_execution_result.get("success", False) is False:
+                    return json.dumps({"error": result.js_execution_result.get("error", "Unknown error")})
+                if "results" in result.js_execution_result:
+                    image_result = result.js_execution_result['results'][0]
+                    if image_result.get("success", False) is False:
+                        return json.dumps({"error": image_result.get("error", "Unknown error")})
+                    return json.dumps(image_result["result"], indent=4)
+
+            # For text search, extract structured data
+            schemas = await self._build_schemas(result.cleaned_html, schema_cache_path)
+            extracted = {
+                key: JsonCssExtractionStrategy(schema=schemas[key]).run(
+                    url=url, sections=[result.html]
+                )
+                for key in schemas
+            }
+            return json.dumps(extracted, indent=4)
+
+    async def _build_schemas(self, html: str, schema_cache_path: str = None) -> Dict[str, Dict]:
+        """Build extraction schemas (organic, top stories, etc.)"""
+        home_dir = get_home_folder() if not schema_cache_path else schema_cache_path
+        os.makedirs(f"{home_dir}/schema", exist_ok=True)
+
+        cleaned_html = optimize_html(html, threshold=100)
+
+        organic_schema = None
+        if os.path.exists(f"{home_dir}/schema/organic_schema.json"):
+            with open(f"{home_dir}/schema/organic_schema.json", "r") as f:
+                organic_schema = json.load(f)
+        else:
+            organic_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "snippet": "...",
+            "date": "1 hour ago",
+        }""",
+                query="""The given html is the crawled html from Google search result. Please find the schema for organic search item in the given html, I am interested in title, link, snippet text. date."""
+            )
+
+            with open(f"{home_dir}/schema/organic_schema.json", "w") as f:
+                f.write(json.dumps(organic_schema))
+
+        top_stories_schema = None
+        if os.path.exists(f"{home_dir}/schema/top_stories_schema.json"):
+            with open(f"{home_dir}/schema/top_stories_schema.json", "r") as f:
+                top_stories_schema = json.load(f)
+        else:
+            top_stories_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "title": "...",
+            "link": "...",
+            "source": "Insider Monkey",
+            "date": "1 hour ago",
+        }""",
+                query="""The given html is the crawled html from Google search result. Please find the schema for Top Story item int he given html, I am interested in title, link, source. date and imageUrl."""
+            )
+
+            with open(f"{home_dir}/schema/top_stories_schema.json", "w") as f:
+                f.write(json.dumps(top_stories_schema))
+
+        suggested_query_schema = None
+        if os.path.exists(f"{home_dir}/schema/suggested_query_schema.json"):
+            with open(f"{home_dir}/schema/suggested_query_schema.json", "r") as f:
+                suggested_query_schema = json.load(f)
+        else:
+            suggested_query_schema = JsonCssExtractionStrategy.generate_schema(
+                html=cleaned_html,
+                target_json_example="""{
+            "query": "A for Apple",
+        }""",
+                query="""The given HTML contains the crawled HTML from Google search results. Please find the schema for each suggested query in the section "People also search for" within the given HTML. I am interested in the queries only."""
+            )
+            with open(f"{home_dir}/schema/suggested_query_schema.json", "w") as f:
+                f.write(json.dumps(suggested_query_schema))
+
+        return {
+            "organic_schema": organic_schema,
+            "top_stories_schema": top_stories_schema,
+            "suggested_query_schema": suggested_query_schema,
+        }

crawl4ai/crawlers/google_search/script.js

@@ -0,0 +1,115 @@
+(() => {
+    // Function to extract image data from Google Images page
+    function extractImageData() {
+        const keys = Object.keys(window.W_jd);
+        let allImageData = [];
+        let currentPosition = 0;
+
+        // Get the symbol we'll use (from first valid entry)
+        let targetSymbol;
+        for (let key of keys) {
+            try {
+                const symbols = Object.getOwnPropertySymbols(window.W_jd[key]);
+                if (symbols.length > 0) {
+                    targetSymbol = symbols[0];
+                    break;
+                }
+            } catch (e) {
+                continue;
+            }
+        }
+
+        if (!targetSymbol) return [];
+
+        // Iterate through ALL keys
+        for (let key of keys) {
+            try {
+                const o1 = window.W_jd[key][targetSymbol]
+                if (!o1) continue;
+                const data = Object.values(o1)[0]
+                // const data = window.W_jd[key][targetSymbol]?.Ws;
+                // Check if this is a valid image data entry
+                if (data && Array.isArray(data[1])) {
+                    const processedData = processImageEntry(data, currentPosition);
+                    if (processedData) {
+                        allImageData.push(processedData);
+                        currentPosition++;
+                    }
+                }
+            } catch (e) {
+                continue;
+            }
+        }
+
+        return allImageData;
+    }
+
+    function processImageEntry(entry, position) {
+        const imageData = entry[1];
+        if (!Array.isArray(imageData)) return null;
+
+        // Extract the image ID
+        const imageId = imageData[1];
+        if (!imageId) return null;
+
+        // Find the corresponding DOM element
+        const domElement = document.querySelector(`[data-docid="${imageId}"]`);
+        if (!domElement) return null;
+
+        // Extract data from the array structure
+        const [
+            _,
+            id,
+            thumbnailInfo,
+            imageInfo,
+            __,
+            ___,
+            rgb,
+            ____,
+            _____,
+            metadata
+        ] = imageData;
+
+        // Ensure we have the required data
+        if (!thumbnailInfo || !imageInfo) return null;
+
+        // Extract metadata from DOM
+        const title = domElement?.querySelector('.toI8Rb')?.textContent?.trim();
+        const source = domElement?.querySelector('.guK3rf')?.textContent?.trim();
+        const link = domElement?.querySelector('a.EZAeBe')?.href;
+
+        if (!link) return null;
+
+        // Build Google Image URL
+        const googleUrl = buildGoogleImageUrl(imageInfo[0], link, imageId, imageInfo[1], imageInfo[2]);
+
+        return {
+            title,
+            imageUrl: imageInfo[0],
+            imageWidth: imageInfo[2],
+            imageHeight: imageInfo[1],
+            thumbnailUrl: thumbnailInfo[0],
+            thumbnailWidth: thumbnailInfo[2],
+            thumbnailHeight: thumbnailInfo[1],
+            source,
+            domain: metadata['2000']?.[1] || new URL(link).hostname,
+            link,
+            googleUrl,
+            position: position + 1
+        };
+    }
+
+    function buildGoogleImageUrl(imgUrl, refUrl, tbnid, height, width) {
+        const params = new URLSearchParams({
+            imgurl: imgUrl,
+            tbnid: tbnid,
+            imgrefurl: refUrl,
+            docid: tbnid,
+            w: width.toString(),
+            h: height.toString(),
+        });
+
+        return `https://www.google.com/imgres?${params.toString()}`;
+    }
+    return extractImageData();
+})();

crawl4ai/deep_crawling/__init__.py

@@ -0,0 +1,47 @@
+# deep_crawling/__init__.py
+from .base_strategy import DeepCrawlDecorator, DeepCrawlStrategy
+from .bfs_strategy import BFSDeepCrawlStrategy
+from .bff_strategy import BestFirstCrawlingStrategy
+from .dfs_strategy import DFSDeepCrawlStrategy
+from .filters import (
+    FilterChain,
+    ContentTypeFilter,
+    DomainFilter,
+    URLFilter,
+    URLPatternFilter,
+    FilterStats,
+    ContentRelevanceFilter,
+    SEOFilter
+)
+from .scorers import (
+    KeywordRelevanceScorer,
+    URLScorer,
+    CompositeScorer,
+    DomainAuthorityScorer,
+    FreshnessScorer,
+    PathDepthScorer,
+    ContentTypeScorer
+)
+
+__all__ = [
+    "DeepCrawlDecorator",
+    "DeepCrawlStrategy",
+    "BFSDeepCrawlStrategy",
+    "BestFirstCrawlingStrategy",
+    "DFSDeepCrawlStrategy",
+    "FilterChain",
+    "ContentTypeFilter",
+    "DomainFilter",
+    "URLFilter",
+    "URLPatternFilter",
+    "FilterStats",
+    "ContentRelevanceFilter",
+    "SEOFilter",
+    "KeywordRelevanceScorer",
+    "URLScorer",
+    "CompositeScorer",
+    "DomainAuthorityScorer",
+    "FreshnessScorer",
+    "PathDepthScorer",
+    "ContentTypeScorer",
+]

crawl4ai/deep_crawling/base_strategy.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator, Optional, Set, List, Dict
+from functools import wraps
+from contextvars import ContextVar
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+
+
+class DeepCrawlDecorator:
+    """Decorator that adds deep crawling capability to arun method."""
+    deep_crawl_active = ContextVar("deep_crawl_active", default=False)
+    
+    def __init__(self, crawler: AsyncWebCrawler): 
+        self.crawler = crawler
+
+    def __call__(self, original_arun):
+        @wraps(original_arun)
+        async def wrapped_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
+            # If deep crawling is already active, call the original method to avoid recursion.
+            if config and config.deep_crawl_strategy and not self.deep_crawl_active.get():
+                token = self.deep_crawl_active.set(True)
+                # Await the arun call to get the actual result object.
+                result_obj = await config.deep_crawl_strategy.arun(
+                    crawler=self.crawler,
+                    start_url=url,
+                    config=config
+                )
+                if config.stream:
+                    async def result_wrapper():
+                        try:
+                            async for result in result_obj:
+                                yield result
+                        finally:
+                            self.deep_crawl_active.reset(token)
+                    return result_wrapper()
+                else:
+                    try:
+                        return result_obj
+                    finally:
+                        self.deep_crawl_active.reset(token)
+            return await original_arun(url, config=config, **kwargs)
+        return wrapped_arun
+
+class DeepCrawlStrategy(ABC):
+    """
+    Abstract base class for deep crawling strategies.
+    
+    Core functions:
+      - arun: Main entry point that returns an async generator of CrawlResults.
+      - shutdown: Clean up resources.
+      - can_process_url: Validate a URL and decide whether to process it.
+      - _process_links: Extract and process links from a CrawlResult.
+    """
+
+    @abstractmethod
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) mode:
+        Processes one BFS level at a time, then yields all the results.
+        """
+        pass
+
+    @abstractmethod
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming mode:
+        Processes one BFS level at a time and yields results immediately as they arrive.
+        """
+        pass
+    
+    async def arun(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: Optional[CrawlerRunConfig] = None,
+    ) -> RunManyReturn:
+        """
+        Traverse the given URL using the specified crawler.
+        
+        Args:
+            start_url (str): The URL from which to start crawling.
+            crawler (AsyncWebCrawler): The crawler instance to use.
+            crawler_run_config (Optional[CrawlerRunConfig]): Crawler configuration.
+        
+        Returns:
+            Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+        """
+        if config is None:
+            raise ValueError("CrawlerRunConfig must be provided")
+
+        if config.stream:
+            return self._arun_stream(start_url, crawler, config)
+        else:
+            return await self._arun_batch(start_url, crawler, config)
+
+    def __call__(self, start_url: str, crawler: AsyncWebCrawler, config: CrawlerRunConfig):
+        return self.arun(start_url, crawler, config)
+
+    @abstractmethod
+    async def shutdown(self) -> None:
+        """
+        Clean up resources used by the deep crawl strategy.
+        """
+        pass
+
+    @abstractmethod
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validate the URL format and apply custom filtering logic.
+        
+        Args:
+            url (str): The URL to validate.
+            depth (int): The current depth in the crawl.
+        
+        Returns:
+            bool: True if the URL should be processed, False otherwise.
+        """
+        pass
+
+    @abstractmethod
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_level: List[tuple],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extract and process links from the given crawl result.
+        
+        This method should:
+          - Validate each extracted URL using can_process_url.
+          - Optionally score URLs.
+          - Append valid URLs (and their parent references) to the next_level list.
+          - Update the depths dictionary with the new depth for each URL.
+        
+        Args:
+            result (CrawlResult): The result from a crawl operation.
+            source_url (str): The URL from which this result was obtained.
+            current_depth (int): The depth at which the source URL was processed.
+            visited (Set[str]): Set of already visited URLs.
+            next_level (List[tuple]): List of tuples (url, parent_url) for the next BFS level.
+            depths (Dict[str, int]): Mapping of URLs to their current depth.
+        """
+        pass
+

crawl4ai/deep_crawling/bff_strategy.py

@@ -0,0 +1,255 @@
+# best_first_crawling_strategy.py
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+from urllib.parse import urlparse
+
+from ..models import TraversalStats
+from .filters import FilterChain
+from .scorers import URLScorer
+from . import DeepCrawlStrategy
+
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
+
+from math import inf as infinity
+
+# Configurable batch size for processing items from the priority queue
+BATCH_SIZE = 10
+
+
+class BestFirstCrawlingStrategy(DeepCrawlStrategy):
+    """
+    Best-First Crawling Strategy using a priority queue.
+    
+    This strategy prioritizes URLs based on their score, ensuring that higher-value
+    pages are crawled first. It reimplements the core traversal loop to use a priority
+    queue while keeping URL validation and link discovery consistent with our design.
+    
+    Core methods:
+      - arun: Returns either a list (batch mode) or an async generator (stream mode).
+      - _arun_best_first: Core generator that uses a priority queue to yield CrawlResults.
+      - can_process_url: Validates URLs and applies filtering (inherited behavior).
+      - link_discovery: Extracts and validates links from a CrawlResult.
+    """
+    def __init__(
+        self,
+        max_depth: int,
+        filter_chain: FilterChain = FilterChain(),
+        url_scorer: Optional[URLScorer] = None,
+        include_external: bool = False,
+        max_pages: int = infinity,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.url_scorer = url_scorer
+        self.include_external = include_external
+        self.max_pages = max_pages
+        self.logger = logger or logging.getLogger(__name__)
+        self.stats = TraversalStats(start_time=datetime.now())
+        self._cancel_event = asyncio.Event()
+        self._pages_crawled = 0
+
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validate the URL format and apply filtering.
+        For the starting URL (depth 0), filtering is bypassed.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError("Missing scheme or netloc")
+            if parsed.scheme not in ("http", "https"):
+                raise ValueError("Invalid scheme")
+            if "." not in parsed.netloc:
+                raise ValueError("Invalid domain")
+        except Exception as e:
+            self.logger.warning(f"Invalid URL: {url}, error: {e}")
+            return False
+
+        if depth != 0 and not await self.filter_chain.apply(url):
+            return False
+
+        return True
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_links: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extract links from the crawl result, validate them, and append new URLs
+        (with their parent references) to next_links.
+        Also updates the depths dictionary.
+        """
+        new_depth = current_depth + 1
+        if new_depth > self.max_depth:
+            return
+            
+        # If we've reached the max pages limit, don't discover new links
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            return
+
+        # Retrieve internal links; include external links if enabled.
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        # If we have more links than remaining capacity, limit how many we'll process
+        valid_links = []
+        for link in links:
+            url = link.get("href")
+            if url in visited:
+                continue
+            if not await self.can_process_url(url, new_depth):
+                self.stats.urls_skipped += 1
+                continue
+                
+            valid_links.append(url)
+            
+        # If we have more valid links than capacity, limit them
+        if len(valid_links) > remaining_capacity:
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
+            
+        # Record the new depths and add to next_links
+        for url in valid_links:
+            depths[url] = new_depth
+            next_links.append((url, source_url))
+
+    async def _arun_best_first(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Core best-first crawl method using a priority queue.
+        
+        The queue items are tuples of (score, depth, url, parent_url). Lower scores
+        are treated as higher priority. URLs are processed in batches for efficiency.
+        """
+        queue: asyncio.PriorityQueue = asyncio.PriorityQueue()
+        # Push the initial URL with score 0 and depth 0.
+        await queue.put((0, 0, start_url, None))
+        visited: Set[str] = set()
+        depths: Dict[str, int] = {start_url: 0}
+
+        while not queue.empty() and not self._cancel_event.is_set():
+            # Stop if we've reached the max pages limit
+            if self._pages_crawled >= self.max_pages:
+                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+                break
+                
+            batch: List[Tuple[float, int, str, Optional[str]]] = []
+            # Retrieve up to BATCH_SIZE items from the priority queue.
+            for _ in range(BATCH_SIZE):
+                if queue.empty():
+                    break
+                item = await queue.get()
+                score, depth, url, parent_url = item
+                if url in visited:
+                    continue
+                visited.add(url)
+                batch.append(item)
+
+            if not batch:
+                continue
+
+            # Process the current batch of URLs.
+            urls = [item[2] for item in batch]
+            batch_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=urls, config=batch_config)
+            async for result in stream_gen:
+                result_url = result.url
+                # Find the corresponding tuple from the batch.
+                corresponding = next((item for item in batch if item[2] == result_url), None)
+                if not corresponding:
+                    continue
+                score, depth, url, parent_url = corresponding
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent_url
+                result.metadata["score"] = score
+                
+                # Count only successful crawls toward max_pages limit
+                if result.success:
+                    self._pages_crawled += 1
+                
+                yield result
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Discover new links from this result
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, result_url, depth, visited, new_links, depths)
+                    
+                    for new_url, new_parent in new_links:
+                        new_depth = depths.get(new_url, depth + 1)
+                        new_score = self.url_scorer.score(new_url) if self.url_scorer else 0
+                        await queue.put((new_score, new_depth, new_url, new_parent))
+
+        # End of crawl.
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Best-first crawl in batch mode.
+        
+        Aggregates all CrawlResults into a list.
+        """
+        results: List[CrawlResult] = []
+        async for result in self._arun_best_first(start_url, crawler, config):
+            results.append(result)
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Best-first crawl in streaming mode.
+        
+        Yields CrawlResults as they become available.
+        """
+        async for result in self._arun_best_first(start_url, crawler, config):
+            yield result
+
+    async def arun(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: Optional[CrawlerRunConfig] = None,
+    ) -> "RunManyReturn":
+        """
+        Main entry point for best-first crawling.
+        
+        Returns either a list (batch mode) or an async generator (stream mode)
+        of CrawlResults.
+        """
+        if config is None:
+            raise ValueError("CrawlerRunConfig must be provided")
+        if config.stream:
+            return self._arun_stream(start_url, crawler, config)
+        else:
+            return await self._arun_batch(start_url, crawler, config)
+
+    async def shutdown(self) -> None:
+        """
+        Signal cancellation and clean up resources.
+        """
+        self._cancel_event.set()
+        self.stats.end_time = datetime.now()

crawl4ai/deep_crawling/bfs_strategy.py

@@ -0,0 +1,241 @@
+# bfs_deep_crawl_strategy.py
+import asyncio
+import logging
+from datetime import datetime
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+from urllib.parse import urlparse
+
+from ..models import TraversalStats
+from .filters import FilterChain
+from .scorers import URLScorer
+from . import DeepCrawlStrategy  
+from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
+from math import inf as infinity
+
+class BFSDeepCrawlStrategy(DeepCrawlStrategy):
+    """
+    Breadth-First Search deep crawling strategy.
+    
+    Core functions:
+      - arun: Main entry point; splits execution into batch or stream modes.
+      - link_discovery: Extracts, filters, and (if needed) scores the outgoing URLs.
+      - can_process_url: Validates URL format and applies the filter chain.
+    """
+    def __init__(
+        self,
+        max_depth: int,
+        filter_chain: FilterChain = FilterChain(),
+        url_scorer: Optional[URLScorer] = None,        
+        include_external: bool = False,
+        score_threshold: float = -infinity,
+        max_pages: int = infinity,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.url_scorer = url_scorer
+        self.include_external = include_external
+        self.score_threshold = score_threshold
+        self.max_pages = max_pages
+        self.logger = logger or logging.getLogger(__name__)
+        self.stats = TraversalStats(start_time=datetime.now())
+        self._cancel_event = asyncio.Event()
+        self._pages_crawled = 0
+
+    async def can_process_url(self, url: str, depth: int) -> bool:
+        """
+        Validates the URL and applies the filter chain.
+        For the start URL (depth 0) filtering is bypassed.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                raise ValueError("Missing scheme or netloc")
+            if parsed.scheme not in ("http", "https"):
+                raise ValueError("Invalid scheme")
+            if "." not in parsed.netloc:
+                raise ValueError("Invalid domain")
+        except Exception as e:
+            self.logger.warning(f"Invalid URL: {url}, error: {e}")
+            return False
+
+        if depth != 0 and not await self.filter_chain.apply(url):
+            return False
+
+        return True
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        visited: Set[str],
+        next_level: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Extracts links from the crawl result, validates and scores them, and
+        prepares the next level of URLs.
+        Each valid URL is appended to next_level as a tuple (url, parent_url)
+        and its depth is tracked.
+        """            
+        next_depth = current_depth + 1
+        if next_depth > self.max_depth:
+            return
+
+        # If we've reached the max pages limit, don't discover new links
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
+            return
+
+        # Get internal links and, if enabled, external links.
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        valid_links = []
+        
+        # First collect all valid links
+        for link in links:
+            url = link.get("href")
+            if url in visited:
+                continue
+            if not await self.can_process_url(url, next_depth):
+                self.stats.urls_skipped += 1
+                continue
+
+            # Score the URL if a scorer is provided
+            score = self.url_scorer.score(url) if self.url_scorer else 0
+            
+            # Skip URLs with scores below the threshold
+            if score < self.score_threshold:
+                self.logger.debug(f"URL {url} skipped: score {score} below threshold {self.score_threshold}")
+                self.stats.urls_skipped += 1
+                continue
+            
+            valid_links.append((url, score))
+        
+        # If we have more valid links than capacity, sort by score and take the top ones
+        if len(valid_links) > remaining_capacity:
+            if self.url_scorer:
+                # Sort by score in descending order
+                valid_links.sort(key=lambda x: x[1], reverse=True)
+            # Take only as many as we have capacity for
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
+            
+        # Process the final selected links
+        for url, score in valid_links:
+            # attach the score to metadata if needed
+            if score:
+                result.metadata = result.metadata or {}
+                result.metadata["score"] = score
+            next_level.append((url, source_url))
+            depths[url] = next_depth
+
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) mode:
+        Processes one BFS level at a time, then yields all the results.
+        """
+        visited: Set[str] = set()
+        # current_level holds tuples: (url, parent_url)
+        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        results: List[CrawlResult] = []
+
+        while current_level and not self._cancel_event.is_set():
+            next_level: List[Tuple[str, Optional[str]]] = []
+            urls = [url for url, _ in current_level]
+            visited.update(urls)
+
+            # Clone the config to disable deep crawling recursion and enforce batch mode.
+            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
+            batch_results = await crawler.arun_many(urls=urls, config=batch_config)
+            
+            # Update pages crawled counter - count only successful crawls
+            successful_results = [r for r in batch_results if r.success]
+            self._pages_crawled += len(successful_results)
+            
+            for result in batch_results:
+                url = result.url
+                depth = depths.get(url, 0)
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                result.metadata["parent_url"] = parent_url
+                results.append(result)
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Link discovery will handle the max pages limit internally
+                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+
+            current_level = next_level
+
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming mode:
+        Processes one BFS level at a time and yields results immediately as they arrive.
+        """
+        visited: Set[str] = set()
+        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        while current_level and not self._cancel_event.is_set():
+            next_level: List[Tuple[str, Optional[str]]] = []
+            urls = [url for url, _ in current_level]
+            visited.update(urls)
+
+            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=urls, config=stream_config)
+            
+            # Keep track of processed results for this batch
+            results_count = 0
+            async for result in stream_gen:
+                url = result.url
+                depth = depths.get(url, 0)
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                parent_url = next((parent for (u, parent) in current_level if u == url), None)
+                result.metadata["parent_url"] = parent_url
+                
+                # Count only successful crawls
+                if result.success:
+                    self._pages_crawled += 1
+                
+                results_count += 1
+                yield result
+                
+                # Only discover links from successful crawls
+                if result.success:
+                    # Link discovery will handle the max pages limit internally
+                    await self.link_discovery(result, url, depth, visited, next_level, depths)
+            
+            # If we didn't get results back (e.g. due to errors), avoid getting stuck in an infinite loop
+            # by considering these URLs as visited but not counting them toward the max_pages limit
+            if results_count == 0 and urls:
+                self.logger.warning(f"No results returned for {len(urls)} URLs, marking as visited")
+                
+            current_level = next_level
+
+    async def shutdown(self) -> None:
+        """
+        Clean up resources and signal cancellation of the crawl.
+        """
+        self._cancel_event.set()
+        self.stats.end_time = datetime.now()

crawl4ai/deep_crawling/crazy.py

@@ -0,0 +1,432 @@
+from __future__ import annotations
+# I just got crazy, trying to wrute K&R C but in Python. Right now I feel like I'm in a quantum state.
+# I probably won't use this; I just want to leave it here. A century later, the future human race will be like, "WTF?"
+
+# ------ Imports That Will Make You Question Reality ------ #
+from functools import wraps
+from contextvars import ContextVar
+import inspect
+
+from crawl4ai import CacheMode
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai.models import CrawlResult, TraversalStats
+from crawl4ai.deep_crawling.filters import FilterChain
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+import time
+import logging
+from urllib.parse import urlparse
+
+from abc import ABC, abstractmethod
+from collections import deque
+import asyncio
+from typing import (
+    AsyncGenerator,
+    Dict,
+    List,
+    TypeVar,
+    Generic,
+    Tuple,
+    Callable,
+    Awaitable,
+    Union,
+)
+from functools import lru_cache
+import mmh3
+from bitarray import bitarray
+import numpy as np
+from heapq import heappush, heappop
+
+# ------ Type Algebra Mastery ------ #
+CrawlResultT = TypeVar("CrawlResultT", bound="CrawlResult")
+PriorityT = TypeVar("PriorityT")
+P = TypeVar("P")
+
+# ------ Hyperscalar Context Management ------ #
+deep_crawl_ctx = ContextVar("deep_crawl_stack", default=deque())
+
+# ------ Algebraic Crawler Monoid ------ #
+class TraversalContext:
+    __slots__ = ('visited', 'frontier', 'depths', 'priority_fn', 'current_depth')
+    
+    def __init__(self,
+                 priority_fn: Callable[[str], Awaitable[float]] = lambda _: 1.0):
+        self.visited: BloomFilter = BloomFilter(10**6, 0.01)  # 1M items, 1% FP
+        self.frontier: PriorityQueue = PriorityQueue()
+        self.depths: Dict[str, int] = {}
+        self.priority_fn = priority_fn
+        self.current_depth = 0
+
+    def clone_for_level(self) -> TraversalContext:
+        """Monadic context propagation"""
+        new_ctx = TraversalContext(self.priority_fn)
+        new_ctx.visited = self.visited.copy()
+        new_ctx.depths = self.depths.copy()
+        new_ctx.current_depth = self.current_depth
+        return new_ctx
+
+class PriorityQueue(Generic[PriorityT]):
+    """Fibonacci heap-inspired priority queue with O(1) amortized operations"""
+    __slots__ = ('_heap', '_index')
+
+    def __init__(self):
+        self._heap: List[Tuple[PriorityT, float, P]] = []
+        self._index: Dict[P, int] = {}
+
+    def insert(self, priority: PriorityT, item: P) -> None:
+        tiebreaker = time.time()  # Ensure FIFO for equal priorities
+        heappush(self._heap, (priority, tiebreaker, item))
+        self._index[item] = len(self._heap) - 1
+
+    def extract(self, top_n = 1) -> P:
+        items = []
+        for _ in range(top_n):
+            if not self._heap:
+                break
+            priority, _, item = heappop(self._heap)
+            del self._index[item]
+            items.append(item)
+        if not items:
+            raise IndexError("Priority queue empty")
+        return items
+        # while self._heap:
+        #     _, _, item = heappop(self._heap)
+        #     if item in self._index:
+        #         del self._index[item]
+        #         return item
+        raise IndexError("Priority queue empty")
+
+
+    def is_empty(self) -> bool:
+        return not bool(self._heap)
+
+class BloomFilter:
+    """Optimal Bloom filter using murmur3 hash avalanche"""
+    __slots__ = ('size', 'hashes', 'bits')
+
+    def __init__(self, capacity: int, error_rate: float):
+        self.size = self._optimal_size(capacity, error_rate)
+        self.hashes = self._optimal_hashes(capacity, self.size)
+        self.bits = bitarray(self.size)
+        self.bits.setall(False)
+
+    @staticmethod
+    def _optimal_size(n: int, p: float) -> int:
+        m = - (n * np.log(p)) / (np.log(2) ** 2)
+        return int(np.ceil(m))
+
+    @staticmethod
+    def _optimal_hashes(n: int, m: int) -> int:
+        k = (m / n) * np.log(2)
+        return int(np.ceil(k))
+
+    def add(self, item: str) -> None:
+        for seed in range(self.hashes):
+            digest = mmh3.hash(item, seed) % self.size
+            self.bits[digest] = True
+
+    def __contains__(self, item: str) -> bool:
+        return all(
+            self.bits[mmh3.hash(item, seed) % self.size]
+            for seed in range(self.hashes)
+        )
+
+    def copy(self) -> BloomFilter:
+        new = object.__new__(BloomFilter)
+        new.size = self.size
+        new.hashes = self.hashes
+        new.bits = self.bits.copy()
+        return new
+    
+    def __len__(self) -> int:
+        """
+        Estimates the number of items in the filter using the 
+        count of set bits and the formula:
+        n = -m/k * ln(1 - X/m)
+        where:
+            m = size of bit array
+            k = number of hash functions
+            X = count of set bits
+        """
+        set_bits = self.bits.count(True)
+        if set_bits == 0:
+            return 0
+            
+        # Use the inverse bloom filter formula to estimate cardinality
+        return int(
+            -(self.size / self.hashes) * 
+            np.log(1 - set_bits / self.size)
+        )
+    
+    def bit_count(self) -> int:
+        """Returns the raw count of set bits in the filter"""
+        return self.bits.count(True)
+        
+    def __repr__(self) -> str:
+        return f"BloomFilter(est_items={len(self)}, bits={self.bit_count()}/{self.size})"
+
+# ------ Hyper-Optimal Deep Crawl Core ------ #
+class DeepCrawlDecorator:
+    """Metaprogramming marvel: Zero-cost deep crawl abstraction"""
+    def __init__(self, crawler: AsyncWebCrawler):
+        self.crawler = crawler
+
+    def __call__(self, original_arun: Callable) -> Callable:
+        @wraps(original_arun)
+        async def quantum_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
+            stack = deep_crawl_ctx.get()
+            if config and config.deep_crawl_strategy and not stack:
+                stack.append(self.crawler)
+                try:
+                    deep_crawl_ctx.set(stack)
+                    async for result in config.deep_crawl_strategy.traverse(
+                        start_url=url,
+                        crawler=self.crawler,
+                        config=config
+                    ):
+                        yield result
+                finally:
+                    stack.pop()
+                    deep_crawl_ctx.set(stack)
+            else:
+                result = await original_arun(url, config=config, **kwargs)
+                yield result
+        return quantum_arun
+
+
+async def collect_results(url, crawler, config):
+    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
+        setattr(crawler, "arun", getattr(crawler, "original_arun"))
+
+    ret = crawler.arun(url, config=config)
+    # If arun is an async generator, iterate over it
+    if inspect.isasyncgen(ret):
+        return [r async for r in ret]
+    # Otherwise, await the coroutine and normalize to a list
+    result = await ret
+    return result if isinstance(result, list) else [result]
+
+async def collect_many_results(url, crawler, config):
+    # Replace back arun to its original implementation
+    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
+        setattr(crawler, "arun", getattr(crawler, "original_arun"))
+    ret = crawler.arun_many(url, config=config)
+    # If arun is an async generator, iterate over it
+    if inspect.isasyncgen(ret):
+        return [r async for r in ret]
+    # Otherwise, await the coroutine and normalize to a list
+    result = await ret
+    return result if isinstance(result, list) else [result]
+
+
+# ------ Deep Crawl Strategy Interface ------ #
+CrawlResultT = TypeVar("CrawlResultT", bound=CrawlResult)
+# In batch mode we return List[CrawlResult] and in stream mode an AsyncGenerator.
+RunManyReturn = Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
+
+
+class DeepCrawlStrategy(ABC):
+    """Abstract base class that will make Dijkstra smile"""
+    @abstractmethod
+    async def traverse(self,
+                      start_url: str,
+                      crawler: AsyncWebCrawler,
+                      config: CrawlerRunConfig) -> RunManyReturn:
+        """Traverse with O(1) memory complexity via generator fusion"""
+        ...
+
+    @abstractmethod
+    def precompute_priority(self, url: str) -> Awaitable[float]:
+        """Quantum-inspired priority precomputation"""
+        pass
+
+    @abstractmethod
+    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
+        """Hilbert-curve optimized link generation"""
+        pass
+
+# ------ BFS That Would Make Knuth Proud ------ #
+
+def calculate_quantum_batch_size(
+    depth: int,
+    max_depth: int,
+    frontier_size: int,
+    visited_size: int
+) -> int:
+    """
+    Calculates optimal batch size for URL processing using quantum-inspired mathematical principles.
+    
+    This function implements a sophisticated batch size calculation using:
+    1. Golden Ratio (φ) based scaling for optimal irrationality
+    2. Depth-aware amplitude modulation
+    3. Harmonic series dampening
+    4. Logarithmic growth control
+    5. Dynamic frontier adaptation
+    
+    The formula follows the quantum harmonic oscillator principle:
+        N = ⌈φ^(2d) * log₂(|V|) * H(d)⁻¹ * min(20, |F|/10)⌉
+    where:
+        φ = Golden Ratio ((1 + √5) / 2)
+        d = depth factor (normalized remaining depth)
+        |V| = size of visited set
+        H(d) = d-th harmonic number
+        |F| = frontier size
+    
+    Args:
+        depth (int): Current traversal depth
+        max_depth (int): Maximum allowed depth
+        frontier_size (int): Current size of frontier queue
+        visited_size (int): Number of URLs visited so far
+    
+    Returns:
+        int: Optimal batch size bounded between 1 and 100
+        
+    Mathematical Properties:
+        - Maintains O(log n) growth with respect to visited size
+        - Provides φ-optimal distribution of resources
+        - Ensures quantum-like state transitions between depths
+        - Harmonically dampened to prevent exponential explosion
+    """
+    # Golden ratio φ = (1 + √5) / 2
+    φ = (1 + 5 ** 0.5) / 2
+    
+    # Calculate normalized depth factor [0, 1]
+    depth_factor = (max_depth - depth) / max_depth if depth < max_depth else 0
+    
+    # Compute harmonic number for current depth
+    harmonic = sum(1/k for k in range(1, depth + 2))
+    
+    # Calculate quantum batch size
+    batch_size = int(np.ceil(
+        (φ ** (depth_factor * 2)) *          # Golden ratio scaling
+        np.log2(visited_size + 2) *          # Logarithmic growth factor
+        (1 / harmonic) *                     # Harmonic dampening
+        max(1, min(20, frontier_size / 10))  # Frontier-aware scaling
+    ))
+    
+    # Enforce practical bounds
+    return max(1, min(100, batch_size))
+
+
+class BFSDeepCrawlStrategy(DeepCrawlStrategy):
+    """Breadth-First Search with Einstein-Rosen bridge optimization"""
+    __slots__ = ('max_depth', 'filter_chain', 'priority_fn', 'stats', '_cancel')
+
+    def __init__(self,
+                 max_depth: int,
+                 filter_chain: FilterChain = FilterChain(),
+                 priority_fn: Callable[[str], Awaitable[float]] = lambda url: 1.0,
+                 logger: logging.Logger = None):
+        self.max_depth = max_depth
+        self.filter_chain = filter_chain
+        self.priority_fn = priority_fn
+        self.stats = TraversalStats()
+        self._cancel = asyncio.Event()
+        self.semaphore = asyncio.Semaphore(1000)
+
+    async def traverse(self,
+                      start_url: str,
+                      crawler: AsyncWebCrawler,
+                      config: CrawlerRunConfig) -> RunManyReturn:
+        """Non-blocking BFS with O(b^d) time complexity awareness"""
+        ctx = TraversalContext(self.priority_fn)
+        ctx.frontier.insert(self.priority_fn(start_url), (start_url, None, 0))
+        ctx.visited.add(start_url)
+        ctx.depths[start_url] = 0
+
+        while not ctx.frontier.is_empty() and not self._cancel.is_set():
+            # Use the best algorith, to find top_n value
+            top_n = calculate_quantum_batch_size(
+                depth=ctx.current_depth,
+                max_depth=self.max_depth,
+                frontier_size=len(ctx.frontier._heap),
+                visited_size=len(ctx.visited)
+            )
+
+            urls = ctx.frontier.extract(top_n=top_n)
+            # url, parent, depth = ctx.frontier.extract(top_n=top_n)
+            if urls:
+                ctx.current_depth = urls[0][2]
+
+            async with self.semaphore:
+                results = await collect_many_results([url for (url, parent, depth) in urls], crawler, config)
+                # results = await asyncio.gather(*[
+                #     collect_results(url, crawler, config) for (url, parent, depth) in urls
+                # ])
+                # result = _result[0]
+                for ix, result in enumerate(results):
+                    url, parent, depth = result.url, urls[ix][1], urls[ix][2]
+                    result.metadata['depth'] = depth
+                    result.metadata['parent'] = parent
+                    yield result
+
+                    if depth < self.max_depth:
+                        async for link in self.link_hypercube(result):
+                            if link not in ctx.visited:
+                                priority = self.priority_fn(link)
+                                ctx.frontier.insert(priority, (link, url, depth + 1))
+                                ctx.visited.add(link)
+                                ctx.depths[link] = depth + 1
+
+    @lru_cache(maxsize=65536)
+    async def validate_url(self, url: str) -> bool:
+        """Memoized URL validation with λ-calculus purity"""
+        try:
+            parsed = urlparse(url)
+            return (parsed.scheme in {'http', 'https'}
+                    and '.' in parsed.netloc
+                    and await self.filter_chain.apply(url))
+        except Exception:
+            return False
+
+    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
+        """Hilbert-ordered link generation with O(1) yield latency"""
+        links = (link['href'] for link in result.links.get('internal', []))
+        validated = filter(self.validate_url, links)
+        for link in sorted(validated, key=lambda x: -self.priority_fn(x)):
+            yield link
+
+    def __aiter__(self) -> AsyncGenerator[CrawlResult, None]:
+        """Native async iterator interface"""
+        return self.traverse()
+
+    async def __anext__(self) -> CrawlResult:
+        """True async iterator protocol implementation"""
+        result = await self.traverse().__anext__()
+        if result:
+            return result
+        raise StopAsyncIteration
+
+    async def precompute_priority(self, url):
+        return super().precompute_priority(url)
+
+    async def shutdown(self):
+        self._cancel.set()
+
+# ------ Usage That Will Drop Jaws ------ #
+async def main():
+    """Quantum crawl example"""
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=2,
+        priority_fn=lambda url: 1.0 / (len(url) + 1e-9),  # Inverse length priority
+        # filter_chain=FilterChain(...)
+    )
+
+    config: CrawlerRunConfig = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        stream=False,
+        verbose=True,
+        cache_mode=CacheMode.BYPASS
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        run_decorator = DeepCrawlDecorator(crawler)
+        setattr(crawler, "original_arun", crawler.arun)
+        crawler.arun = run_decorator(crawler.arun)
+        start_time = time.perf_counter()
+        async for result in crawler.arun("https://docs.crawl4ai.com", config=config):
+            print(f"🌀 {result.url} (Depth: {result.metadata['depth']})")
+        print(f"Deep crawl completed in {time.perf_counter() - start_time:.2f}s")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

crawl4ai/deep_crawling/dfs_strategy.py

@@ -0,0 +1,102 @@
+# dfs_deep_crawl_strategy.py
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+
+from ..models import CrawlResult
+from .bfs_strategy import BFSDeepCrawlStrategy  # noqa
+from ..types import AsyncWebCrawler, CrawlerRunConfig
+
+class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
+    """
+    Depth-First Search (DFS) deep crawling strategy.
+
+    Inherits URL validation and link discovery from BFSDeepCrawlStrategy.
+    Overrides _arun_batch and _arun_stream to use a stack (LIFO) for DFS traversal.
+    """
+    async def _arun_batch(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> List[CrawlResult]:
+        """
+        Batch (non-streaming) DFS mode.
+        Uses a stack to traverse URLs in DFS order, aggregating CrawlResults into a list.
+        """
+        visited: Set[str] = set()
+        # Stack items: (url, parent_url, depth)
+        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+        depths: Dict[str, int] = {start_url: 0}
+        results: List[CrawlResult] = []
+
+        while stack and not self._cancel_event.is_set():
+            url, parent, depth = stack.pop()
+            if url in visited or depth > self.max_depth:
+                continue
+            visited.add(url)
+
+            # Clone config to disable recursive deep crawling.
+            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
+            url_results = await crawler.arun_many(urls=[url], config=batch_config)
+            
+            for result in url_results:
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent
+                if self.url_scorer:
+                    result.metadata["score"] = self.url_scorer.score(url)
+                results.append(result)
+                
+                # Count only successful crawls toward max_pages limit
+                if result.success:
+                    self._pages_crawled += 1
+                    
+                    # Only discover links from successful crawls
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    
+                    # Push new links in reverse order so the first discovered is processed next.
+                    for new_url, new_parent in reversed(new_links):
+                        new_depth = depths.get(new_url, depth + 1)
+                        stack.append((new_url, new_parent, new_depth))
+        return results
+
+    async def _arun_stream(
+        self,
+        start_url: str,
+        crawler: AsyncWebCrawler,
+        config: CrawlerRunConfig,
+    ) -> AsyncGenerator[CrawlResult, None]:
+        """
+        Streaming DFS mode.
+        Uses a stack to traverse URLs in DFS order and yields CrawlResults as they become available.
+        """
+        visited: Set[str] = set()
+        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+        depths: Dict[str, int] = {start_url: 0}
+
+        while stack and not self._cancel_event.is_set():
+            url, parent, depth = stack.pop()
+            if url in visited or depth > self.max_depth:
+                continue
+            visited.add(url)
+
+            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
+            stream_gen = await crawler.arun_many(urls=[url], config=stream_config)
+            async for result in stream_gen:
+                result.metadata = result.metadata or {}
+                result.metadata["depth"] = depth
+                result.metadata["parent_url"] = parent
+                if self.url_scorer:
+                    result.metadata["score"] = self.url_scorer.score(url)
+                yield result
+
+                # Only count successful crawls toward max_pages limit
+                # and only discover links from successful crawls
+                if result.success:
+                    self._pages_crawled += 1
+                    
+                    new_links: List[Tuple[str, Optional[str]]] = []
+                    await self.link_discovery(result, url, depth, visited, new_links, depths)
+                    for new_url, new_parent in reversed(new_links):
+                        new_depth = depths.get(new_url, depth + 1)
+                        stack.append((new_url, new_parent, new_depth))

crawl4ai/deep_crawling/filters.py

@@ -0,0 +1,648 @@
+from abc import ABC, abstractmethod
+from typing import List, Pattern, Set, Union
+from urllib.parse import urlparse
+from array import array
+import re
+import logging
+from functools import lru_cache
+import fnmatch
+from dataclasses import dataclass
+import weakref
+import math
+from collections import defaultdict
+from typing import Dict
+from ..utils import HeadPeekr
+import asyncio
+import inspect
+
+
+@dataclass
+class FilterStats:
+    __slots__ = ("_counters",)
+
+    def __init__(self):
+        # Use array of unsigned ints for atomic operations
+        self._counters = array("I", [0, 0, 0])  # total, passed, rejected
+
+    @property
+    def total_urls(self):
+        return self._counters[0]
+
+    @property
+    def passed_urls(self):
+        return self._counters[1]
+
+    @property
+    def rejected_urls(self):
+        return self._counters[2]
+
+
+class URLFilter(ABC):
+    """Optimized base filter class"""
+
+    __slots__ = ("name", "stats", "_logger_ref")
+
+    def __init__(self, name: str = None):
+        self.name = name or self.__class__.__name__
+        self.stats = FilterStats()
+        # Lazy logger initialization using weakref
+        self._logger_ref = None
+
+    @property
+    def logger(self):
+        if self._logger_ref is None or self._logger_ref() is None:
+            logger = logging.getLogger(f"urlfilter.{self.name}")
+            self._logger_ref = weakref.ref(logger)
+        return self._logger_ref()
+
+    @abstractmethod
+    def apply(self, url: str) -> bool:
+        pass
+
+    def _update_stats(self, passed: bool):
+        # Use direct array index for speed
+        self.stats._counters[0] += 1  # total
+        self.stats._counters[1] += passed  # passed
+        self.stats._counters[2] += not passed  # rejected
+
+
+class FilterChain:
+    """Optimized filter chain"""
+
+    __slots__ = ("filters", "stats", "_logger_ref")
+
+    def __init__(self, filters: List[URLFilter] = None):
+        self.filters = tuple(filters or [])  # Immutable tuple for speed
+        self.stats = FilterStats()
+        self._logger_ref = None
+
+    @property
+    def logger(self):
+        if self._logger_ref is None or self._logger_ref() is None:
+            logger = logging.getLogger("urlfilter.chain")
+            self._logger_ref = weakref.ref(logger)
+        return self._logger_ref()
+
+    def add_filter(self, filter_: URLFilter) -> "FilterChain":
+        """Add a filter to the chain"""
+        self.filters.append(filter_)
+        return self  # Enable method chaining
+
+    async def apply(self, url: str) -> bool:
+        """Apply all filters concurrently when possible"""
+        self.stats._counters[0] += 1  # Total processed URLs
+
+        tasks = []
+        for f in self.filters:
+            result = f.apply(url)
+
+            if inspect.isawaitable(result):
+                tasks.append(result)  # Collect async tasks
+            elif not result:  # Sync rejection
+                self.stats._counters[2] += 1  # Sync rejected
+                return False
+
+        if tasks:
+            results = await asyncio.gather(*tasks)
+
+            # Count how many filters rejected
+            rejections = results.count(False)
+            self.stats._counters[2] += rejections
+
+            if not all(results):
+                return False  # Stop early if any filter rejected
+
+        self.stats._counters[1] += 1  # Passed
+        return True
+
+
+class URLPatternFilter(URLFilter):
+    """Pattern filter balancing speed and completeness"""
+
+    __slots__ = (
+        "_simple_suffixes",
+        "_simple_prefixes",
+        "_domain_patterns",
+        "_path_patterns",
+    )
+
+    PATTERN_TYPES = {
+        "SUFFIX": 1,  # *.html
+        "PREFIX": 2,  # /foo/*
+        "DOMAIN": 3,  # *.example.com
+        "PATH": 4,  # Everything else
+        "REGEX": 5,
+    }
+
+    def __init__(
+        self,
+        patterns: Union[str, Pattern, List[Union[str, Pattern]]],
+        use_glob: bool = True,
+    ):
+        super().__init__()
+        patterns = [patterns] if isinstance(patterns, (str, Pattern)) else patterns
+
+        self._simple_suffixes = set()
+        self._simple_prefixes = set()
+        self._domain_patterns = []
+        self._path_patterns = []
+
+        for pattern in patterns:
+            pattern_type = self._categorize_pattern(pattern)
+            self._add_pattern(pattern, pattern_type)
+
+    def _categorize_pattern(self, pattern: str) -> int:
+        """Categorize pattern for specialized handling"""
+        if not isinstance(pattern, str):
+            return self.PATTERN_TYPES["PATH"]
+
+        # Check if it's a regex pattern
+        if pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern:
+            return self.PATTERN_TYPES["REGEX"]
+
+        if pattern.count("*") == 1:
+            if pattern.startswith("*."):
+                return self.PATTERN_TYPES["SUFFIX"]
+            if pattern.endswith("/*"):
+                return self.PATTERN_TYPES["PREFIX"]
+
+        if "://" in pattern and pattern.startswith("*."):
+            return self.PATTERN_TYPES["DOMAIN"]
+
+        return self.PATTERN_TYPES["PATH"]
+
+    def _add_pattern(self, pattern: str, pattern_type: int):
+        """Add pattern to appropriate matcher"""
+        if pattern_type == self.PATTERN_TYPES["REGEX"]:
+            # For regex patterns, compile directly without glob translation
+            if isinstance(pattern, str) and (
+                pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern
+            ):
+                self._path_patterns.append(re.compile(pattern))
+                return
+        elif pattern_type == self.PATTERN_TYPES["SUFFIX"]:
+            self._simple_suffixes.add(pattern[2:])
+        elif pattern_type == self.PATTERN_TYPES["PREFIX"]:
+            self._simple_prefixes.add(pattern[:-2])
+        elif pattern_type == self.PATTERN_TYPES["DOMAIN"]:
+            self._domain_patterns.append(re.compile(pattern.replace("*.", r"[^/]+\.")))
+        else:
+            if isinstance(pattern, str):
+                # Handle complex glob patterns
+                if "**" in pattern:
+                    pattern = pattern.replace("**", ".*")
+                if "{" in pattern:
+                    # Convert {a,b} to (a|b)
+                    pattern = re.sub(
+                        r"\{([^}]+)\}",
+                        lambda m: f'({"|".join(m.group(1).split(","))})',
+                        pattern,
+                    )
+                pattern = fnmatch.translate(pattern)
+            self._path_patterns.append(
+                pattern if isinstance(pattern, Pattern) else re.compile(pattern)
+            )
+
+    @lru_cache(maxsize=10000)
+    def apply(self, url: str) -> bool:
+        """Hierarchical pattern matching"""
+        # Quick suffix check (*.html)
+        if self._simple_suffixes:
+            path = url.split("?")[0]
+            if path.split("/")[-1].split(".")[-1] in self._simple_suffixes:
+                self._update_stats(True)
+                return True
+
+        # Domain check
+        if self._domain_patterns:
+            for pattern in self._domain_patterns:
+                if pattern.match(url):
+                    self._update_stats(True)
+                    return True
+
+        # Prefix check (/foo/*)
+        if self._simple_prefixes:
+            path = url.split("?")[0]
+            if any(path.startswith(p) for p in self._simple_prefixes):
+                self._update_stats(True)
+                return True
+
+        # Complex patterns
+        if self._path_patterns:
+            if any(p.search(url) for p in self._path_patterns):
+                self._update_stats(True)
+                return True
+
+        self._update_stats(False)
+        return False
+
+
+class ContentTypeFilter(URLFilter):
+    """Optimized content type filter using fast lookups"""
+
+    __slots__ = ("allowed_types", "_ext_map", "_check_extension")
+
+    # Fast extension to mime type mapping
+    _MIME_MAP = {
+        # Text Formats
+        "txt": "text/plain",
+        "html": "text/html",
+        "htm": "text/html",
+        "xhtml": "application/xhtml+xml",
+        "css": "text/css",
+        "csv": "text/csv",
+        "ics": "text/calendar",
+        "js": "application/javascript",
+        # Images
+        "bmp": "image/bmp",
+        "gif": "image/gif",
+        "jpeg": "image/jpeg",
+        "jpg": "image/jpeg",
+        "png": "image/png",
+        "svg": "image/svg+xml",
+        "tiff": "image/tiff",
+        "ico": "image/x-icon",
+        "webp": "image/webp",
+        # Audio
+        "mp3": "audio/mpeg",
+        "wav": "audio/wav",
+        "ogg": "audio/ogg",
+        "m4a": "audio/mp4",
+        "aac": "audio/aac",
+        # Video
+        "mp4": "video/mp4",
+        "mpeg": "video/mpeg",
+        "webm": "video/webm",
+        "avi": "video/x-msvideo",
+        "mov": "video/quicktime",
+        "flv": "video/x-flv",
+        "wmv": "video/x-ms-wmv",
+        "mkv": "video/x-matroska",
+        # Applications
+        "json": "application/json",
+        "xml": "application/xml",
+        "pdf": "application/pdf",
+        "zip": "application/zip",
+        "gz": "application/gzip",
+        "tar": "application/x-tar",
+        "rar": "application/vnd.rar",
+        "7z": "application/x-7z-compressed",
+        "exe": "application/vnd.microsoft.portable-executable",
+        "msi": "application/x-msdownload",
+        # Fonts
+        "woff": "font/woff",
+        "woff2": "font/woff2",
+        "ttf": "font/ttf",
+        "otf": "font/otf",
+        # Microsoft Office
+        "doc": "application/msword",
+        "dot": "application/msword",
+        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "xls": "application/vnd.ms-excel",
+        "ppt": "application/vnd.ms-powerpoint",
+        "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        # OpenDocument Formats
+        "odt": "application/vnd.oasis.opendocument.text",
+        "ods": "application/vnd.oasis.opendocument.spreadsheet",
+        "odp": "application/vnd.oasis.opendocument.presentation",
+        # Archives
+        "tar.gz": "application/gzip",
+        "tgz": "application/gzip",
+        "bz2": "application/x-bzip2",
+        # Others
+        "rtf": "application/rtf",
+        "apk": "application/vnd.android.package-archive",
+        "epub": "application/epub+zip",
+        "jar": "application/java-archive",
+        "swf": "application/x-shockwave-flash",
+        "midi": "audio/midi",
+        "mid": "audio/midi",
+        "ps": "application/postscript",
+        "ai": "application/postscript",
+        "eps": "application/postscript",
+        # Custom or less common
+        "bin": "application/octet-stream",
+        "dmg": "application/x-apple-diskimage",
+        "iso": "application/x-iso9660-image",
+        "deb": "application/x-debian-package",
+        "rpm": "application/x-rpm",
+        "sqlite": "application/vnd.sqlite3",
+        # Placeholder
+        "unknown": "application/octet-stream",  # Fallback for unknown file types
+    }
+
+    @staticmethod
+    @lru_cache(maxsize=1000)
+    def _extract_extension(url: str) -> str:
+        """Extracts file extension from a URL."""
+        # Remove scheme (http://, https://) if present
+        if "://" in url:
+            url = url.split("://", 1)[-1]  # Get everything after '://'
+
+        # Remove domain (everything up to the first '/')
+        path_start = url.find("/")
+        path = url[path_start:] if path_start != -1 else ""
+
+        # Extract last filename in path
+        filename = path.rsplit("/", 1)[-1] if "/" in path else ""
+
+        # Extract and validate extension
+        if "." not in filename:
+            return ""
+
+        return filename.rpartition(".")[-1].lower()
+
+    def __init__(
+        self,
+        allowed_types: Union[str, List[str]],
+        check_extension: bool = True,
+        ext_map: Dict[str, str] = _MIME_MAP,
+    ):
+        super().__init__()
+        # Normalize and store as frozenset for fast lookup
+        self.allowed_types = frozenset(
+            t.lower()
+            for t in (
+                allowed_types if isinstance(allowed_types, list) else [allowed_types]
+            )
+        )
+        self._check_extension = check_extension
+
+        # Pre-compute extension map for allowed types
+        self._ext_map = frozenset(
+            ext
+            for ext, mime in self._MIME_MAP.items()
+            if any(allowed in mime for allowed in self.allowed_types)
+        )
+
+    @lru_cache(maxsize=1000)
+    def _check_url_cached(self, url: str) -> bool:
+        """Cached URL checking"""
+        if not self._check_extension:
+            return True
+        ext = self._extract_extension(url)
+        if not ext:
+            return True
+
+        return ext in self._ext_map
+
+    def apply(self, url: str) -> bool:
+        """Fast extension check with caching"""
+        result = self._check_url_cached(url)
+        self._update_stats(result)
+        return result
+
+
+class DomainFilter(URLFilter):
+    """Optimized domain filter with fast lookups and caching"""
+
+    __slots__ = ("_allowed_domains", "_blocked_domains", "_domain_cache")
+
+    # Regex for fast domain extraction
+    _DOMAIN_REGEX = re.compile(r"://([^/]+)")
+
+    def __init__(
+        self,
+        allowed_domains: Union[str, List[str]] = None,
+        blocked_domains: Union[str, List[str]] = None,
+    ):
+        super().__init__()
+
+        # Convert inputs to frozensets for immutable, fast lookups
+        self._allowed_domains = (
+            frozenset(self._normalize_domains(allowed_domains))
+            if allowed_domains
+            else None
+        )
+        self._blocked_domains = (
+            frozenset(self._normalize_domains(blocked_domains))
+            if blocked_domains
+            else frozenset()
+        )
+
+    @staticmethod
+    def _normalize_domains(domains: Union[str, List[str]]) -> Set[str]:
+        """Fast domain normalization"""
+        if isinstance(domains, str):
+            return {domains.lower()}
+        return {d.lower() for d in domains}
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _extract_domain(url: str) -> str:
+        """Ultra-fast domain extraction with regex and caching"""
+        match = DomainFilter._DOMAIN_REGEX.search(url)
+        return match.group(1).lower() if match else ""
+
+    def apply(self, url: str) -> bool:
+        """Optimized domain checking with early returns"""
+        # Skip processing if no filters
+        if not self._blocked_domains and self._allowed_domains is None:
+            self._update_stats(True)
+            return True
+
+        domain = self._extract_domain(url)
+
+        # Early return for blocked domains
+        if domain in self._blocked_domains:
+            self._update_stats(False)
+            return False
+
+        # If no allowed domains specified, accept all non-blocked
+        if self._allowed_domains is None:
+            self._update_stats(True)
+            return True
+
+        # Final allowed domains check
+        result = domain in self._allowed_domains
+        self._update_stats(result)
+        return result
+
+
+class ContentRelevanceFilter(URLFilter):
+    """BM25-based relevance filter using head section content"""
+
+    __slots__ = ("query_terms", "threshold", "k1", "b", "avgdl")
+
+    def __init__(
+        self,
+        query: str,
+        threshold: float,
+        k1: float = 1.2,
+        b: float = 0.75,
+        avgdl: int = 1000,
+    ):
+        super().__init__(name="BM25RelevanceFilter")
+        self.query_terms = self._tokenize(query)
+        self.threshold = threshold
+        self.k1 = k1  # TF saturation parameter
+        self.b = b  # Length normalization parameter
+        self.avgdl = avgdl  # Average document length (empirical value)
+
+    async def apply(self, url: str) -> bool:
+        head_content = await HeadPeekr.peek_html(url)
+        if not head_content:
+            self._update_stats(False)
+            return False
+
+        # Field extraction with weighting
+        fields = {
+            "title": HeadPeekr.get_title(head_content) or "",
+            "meta": HeadPeekr.extract_meta_tags(head_content),
+        }
+        doc_text = self._build_document(fields)
+
+        score = self._bm25(doc_text)
+        decision = score >= self.threshold
+        self._update_stats(decision)
+        return decision
+
+    def _build_document(self, fields: Dict) -> str:
+        """Weighted document construction"""
+        return " ".join(
+            [
+                fields["title"] * 3,  # Title weight
+                fields["meta"].get("description", "") * 2,
+                fields["meta"].get("keywords", ""),
+                " ".join(fields["meta"].values()),
+            ]
+        )
+
+    def _tokenize(self, text: str) -> List[str]:
+        """Fast case-insensitive tokenization"""
+        return text.lower().split()
+
+    def _bm25(self, document: str) -> float:
+        """Optimized BM25 implementation for head sections"""
+        doc_terms = self._tokenize(document)
+        doc_len = len(doc_terms)
+        tf = defaultdict(int)
+
+        for term in doc_terms:
+            tf[term] += 1
+
+        score = 0.0
+        for term in set(self.query_terms):
+            term_freq = tf[term]
+            idf = math.log((1 + 1) / (term_freq + 0.5) + 1)  # Simplified IDF
+            numerator = term_freq * (self.k1 + 1)
+            denominator = term_freq + self.k1 * (
+                1 - self.b + self.b * (doc_len / self.avgdl)
+            )
+            score += idf * (numerator / denominator)
+
+        return score
+
+
+class SEOFilter(URLFilter):
+    """Quantitative SEO quality assessment filter using head section analysis"""
+
+    __slots__ = ("threshold", "_weights", "_kw_patterns")
+
+    # Based on SEMrush/Google ranking factors research
+    DEFAULT_WEIGHTS = {
+        "title_length": 0.15,
+        "title_kw": 0.18,
+        "meta_description": 0.12,
+        "canonical": 0.10,
+        "robot_ok": 0.20,  # Most critical factor
+        "schema_org": 0.10,
+        "url_quality": 0.15,
+    }
+
+    def __init__(
+        self,
+        threshold: float = 0.65,
+        keywords: List[str] = None,
+        weights: Dict[str, float] = None,
+    ):
+        super().__init__(name="SEOFilter")
+        self.threshold = threshold
+        self._weights = weights or self.DEFAULT_WEIGHTS
+        self._kw_patterns = (
+            re.compile(
+                r"\b({})\b".format("|".join(map(re.escape, keywords or []))), re.I
+            )
+            if keywords
+            else None
+        )
+
+    async def apply(self, url: str) -> bool:
+        head_content = await HeadPeekr.peek_html(url)
+        if not head_content:
+            self._update_stats(False)
+            return False
+
+        meta = HeadPeekr.extract_meta_tags(head_content)
+        title = HeadPeekr.get_title(head_content) or ""
+        parsed_url = urlparse(url)
+
+        scores = {
+            "title_length": self._score_title_length(title),
+            "title_kw": self._score_keyword_presence(title),
+            "meta_description": self._score_meta_description(
+                meta.get("description", "")
+            ),
+            "canonical": self._score_canonical(meta.get("canonical"), url),
+            "robot_ok": 1.0 if "noindex" not in meta.get("robots", "") else 0.0,
+            "schema_org": self._score_schema_org(head_content),
+            "url_quality": self._score_url_quality(parsed_url),
+        }
+
+        total_score = sum(
+            weight * scores[factor] for factor, weight in self._weights.items()
+        )
+
+        decision = total_score >= self.threshold
+        self._update_stats(decision)
+        return decision
+
+    def _score_title_length(self, title: str) -> float:
+        length = len(title)
+        if 50 <= length <= 60:
+            return 1.0
+        if 40 <= length < 50 or 60 < length <= 70:
+            return 0.7
+        return 0.3  # Poor length
+
+    def _score_keyword_presence(self, text: str) -> float:
+        if not self._kw_patterns:
+            return 0.0
+        matches = len(self._kw_patterns.findall(text))
+        return min(matches * 0.3, 1.0)  # Max 3 matches
+
+    def _score_meta_description(self, desc: str) -> float:
+        length = len(desc)
+        if 140 <= length <= 160:
+            return 1.0
+        return 0.5 if 120 <= length <= 200 else 0.2
+
+    def _score_canonical(self, canonical: str, original: str) -> float:
+        if not canonical:
+            return 0.5  # Neutral score
+        return 1.0 if canonical == original else 0.2
+
+    def _score_schema_org(self, html: str) -> float:
+        # Detect any schema.org markup in head
+        return (
+            1.0
+            if re.search(r'<script[^>]+type=["\']application/ld\+json', html)
+            else 0.0
+        )
+
+    def _score_url_quality(self, parsed_url) -> float:
+        score = 1.0
+        path = parsed_url.path.lower()
+
+        # Penalty factors
+        if len(path) > 80:
+            score *= 0.7
+        if re.search(r"\d{4}", path):
+            score *= 0.8  # Numbers in path
+        if parsed_url.query:
+            score *= 0.6  # URL parameters
+        if "_" in path:
+            score *= 0.9  # Underscores vs hyphens
+
+        return score

crawl4ai/deep_crawling/scorers.py

@@ -0,0 +1,519 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Optional
+from dataclasses import dataclass
+from urllib.parse import urlparse, unquote
+import re
+import logging
+from functools import lru_cache
+from array import array
+import ctypes
+import platform
+PLATFORM = platform.system()
+
+# Pre-computed scores for common year differences
+_SCORE_LOOKUP = [1.0, 0.5, 0.3333333333333333, 0.25]
+
+# Pre-computed scores for common year differences
+_FRESHNESS_SCORES = [
+   1.0,    # Current year
+   0.9,    # Last year
+   0.8,    # 2 years ago
+   0.7,    # 3 years ago
+   0.6,    # 4 years ago
+   0.5,    # 5 years ago
+]
+
+class ScoringStats:
+    __slots__ = ('_urls_scored', '_total_score', '_min_score', '_max_score')
+    
+    def __init__(self):
+        self._urls_scored = 0
+        self._total_score = 0.0
+        self._min_score = None  # Lazy initialization
+        self._max_score = None
+    
+    def update(self, score: float) -> None:
+        """Optimized update with minimal operations"""
+        self._urls_scored += 1
+        self._total_score += score
+        
+        # Lazy min/max tracking - only if actually accessed
+        if self._min_score is not None:
+            if score < self._min_score:
+                self._min_score = score
+        if self._max_score is not None:
+            if score > self._max_score:
+                self._max_score = score
+                
+    def get_average(self) -> float:
+        """Direct calculation instead of property"""
+        return self._total_score / self._urls_scored if self._urls_scored else 0.0
+    
+    def get_min(self) -> float:
+        """Lazy min calculation"""
+        if self._min_score is None:
+            self._min_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
+        return self._min_score
+        
+    def get_max(self) -> float:
+        """Lazy max calculation"""
+        if self._max_score is None:
+            self._max_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
+        return self._max_score
+class URLScorer(ABC):
+    __slots__ = ('_weight', '_stats')
+    
+    def __init__(self, weight: float = 1.0):
+        # Store weight directly as float32 for memory efficiency
+        self._weight = ctypes.c_float(weight).value
+        self._stats = ScoringStats()
+    
+    @abstractmethod
+    def _calculate_score(self, url: str) -> float:
+        """Calculate raw score for URL."""
+        pass
+    
+    def score(self, url: str) -> float:
+        """Calculate weighted score with minimal overhead."""
+        score = self._calculate_score(url) * self._weight
+        self._stats.update(score)
+        return score
+    
+    @property
+    def stats(self):
+        """Access to scoring statistics."""
+        return self._stats
+    
+    @property
+    def weight(self):
+        return self._weight
+
+class CompositeScorer(URLScorer):
+    __slots__ = ('_scorers', '_normalize', '_weights_array', '_score_array')
+    
+    def __init__(self, scorers: List[URLScorer], normalize: bool = True):
+        """Initialize composite scorer combining multiple scoring strategies.
+        
+        Optimized for:
+        - Fast parallel scoring
+        - Memory efficient score aggregation
+        - Quick short-circuit conditions
+        - Pre-allocated arrays
+        
+        Args:
+            scorers: List of scoring strategies to combine
+            normalize: Whether to normalize final score by scorer count
+        """
+        super().__init__(weight=1.0)
+        self._scorers = scorers
+        self._normalize = normalize
+        
+        # Pre-allocate arrays for scores and weights
+        self._weights_array = array('f', [s.weight for s in scorers])
+        self._score_array = array('f', [0.0] * len(scorers))
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate combined score from all scoring strategies.
+        
+        Uses:
+        1. Pre-allocated arrays for scores
+        2. Short-circuit on zero scores
+        3. Optimized normalization
+        4. Vectorized operations where possible
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Combined and optionally normalized score
+        """
+        total_score = 0.0
+        scores = self._score_array
+        
+        # Get scores from all scorers
+        for i, scorer in enumerate(self._scorers):
+            # Use public score() method which applies weight
+            scores[i] = scorer.score(url)
+            total_score += scores[i]
+            
+        # Normalize if requested
+        if self._normalize and self._scorers:
+            count = len(self._scorers)
+            return total_score / count
+            
+        return total_score
+
+    def score(self, url: str) -> float:
+        """Public scoring interface with stats tracking.
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Final combined score
+        """
+        score = self._calculate_score(url)
+        self.stats.update(score)
+        return score
+
+class KeywordRelevanceScorer(URLScorer):
+    __slots__ = ('_weight', '_stats', '_keywords', '_case_sensitive')
+    
+    def __init__(self, keywords: List[str], weight: float = 1.0, case_sensitive: bool = False):
+        super().__init__(weight=weight)
+        self._case_sensitive = case_sensitive
+        # Pre-process keywords once
+        self._keywords = [k if case_sensitive else k.lower() for k in keywords]
+    
+    @lru_cache(maxsize=10000)
+    def _url_bytes(self, url: str) -> bytes:
+        """Cache decoded URL bytes"""
+        return url.encode('utf-8') if self._case_sensitive else url.lower().encode('utf-8')
+    
+    
+    def _calculate_score(self, url: str) -> float:
+        """Fast string matching without regex or byte conversion"""
+        if not self._case_sensitive:
+            url = url.lower()
+            
+        matches = sum(1 for k in self._keywords if k in url)
+        
+        # Fast return paths
+        if not matches:
+            return 0.0
+        if matches == len(self._keywords):
+            return 1.0
+            
+        return matches / len(self._keywords)
+
+class PathDepthScorer(URLScorer):
+    __slots__ = ('_weight', '_stats', '_optimal_depth')  # Remove _url_cache
+    
+    def __init__(self, optimal_depth: int = 3, weight: float = 1.0):
+        super().__init__(weight=weight)
+        self._optimal_depth = optimal_depth
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _quick_depth(path: str) -> int:
+        """Ultra fast path depth calculation.
+        
+        Examples:
+            - "http://example.com" -> 0  # No path segments
+            - "http://example.com/" -> 0  # Empty path
+            - "http://example.com/a" -> 1
+            - "http://example.com/a/b" -> 2
+        """
+        if not path or path == '/':
+            return 0
+            
+        if '/' not in path:
+            return 0
+            
+        depth = 0
+        last_was_slash = True
+        
+        for c in path:
+            if c == '/':
+                if not last_was_slash:
+                    depth += 1
+                last_was_slash = True
+            else:
+                last_was_slash = False
+                
+        if not last_was_slash:
+            depth += 1
+            
+        return depth
+
+    @lru_cache(maxsize=10000)  # Cache the whole calculation
+    def _calculate_score(self, url: str) -> float:
+        pos = url.find('/', url.find('://') + 3)
+        if pos == -1:
+            depth = 0
+        else:
+            depth = self._quick_depth(url[pos:])
+            
+        # Use lookup table for common distances
+        distance = depth - self._optimal_depth
+        distance = distance if distance >= 0 else -distance  # Faster than abs()
+        
+        if distance < 4:
+            return _SCORE_LOOKUP[distance]
+            
+        return 1.0 / (1.0 + distance)                                             
+
+class ContentTypeScorer(URLScorer):
+    __slots__ = ('_weight', '_exact_types', '_regex_types')
+
+    def __init__(self, type_weights: Dict[str, float], weight: float = 1.0):
+        """Initialize scorer with type weights map.
+        
+        Args:
+            type_weights: Dict mapping file extensions/patterns to scores (e.g. {'.html$': 1.0})
+            weight: Overall weight multiplier for this scorer
+        """
+        super().__init__(weight=weight)
+        self._exact_types = {}  # Fast lookup for simple extensions
+        self._regex_types = []  # Fallback for complex patterns
+        
+        # Split into exact vs regex matchers for performance
+        for pattern, score in type_weights.items():
+            if pattern.startswith('.') and pattern.endswith('$'):
+                ext = pattern[1:-1]
+                self._exact_types[ext] = score
+            else:
+                self._regex_types.append((re.compile(pattern), score))
+                
+        # Sort complex patterns by score for early exit
+        self._regex_types.sort(key=lambda x: -x[1])
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _quick_extension(url: str) -> str:
+        """Extract file extension ultra-fast without regex/splits.
+        
+        Handles:
+        - Basic extensions: "example.html" -> "html"
+        - Query strings: "page.php?id=1" -> "php" 
+        - Fragments: "doc.pdf#page=1" -> "pdf"
+        - Path params: "file.jpg;width=100" -> "jpg"
+        
+        Args:
+            url: URL to extract extension from
+            
+        Returns:
+            Extension without dot, or empty string if none found
+        """
+        pos = url.rfind('.')
+        if pos == -1:
+            return ''
+        
+        # Find first non-alphanumeric char after extension
+        end = len(url)
+        for i in range(pos + 1, len(url)):
+            c = url[i]
+            # Stop at query string, fragment, path param or any non-alphanumeric
+            if c in '?#;' or not c.isalnum():
+                end = i
+                break
+                
+        return url[pos + 1:end].lower()
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate content type score for URL.
+        
+        Uses staged approach:
+        1. Try exact extension match (fast path)
+        2. Fall back to regex patterns if needed
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Score between 0.0 and 1.0 * weight
+        """
+        # Fast path: direct extension lookup
+        ext = self._quick_extension(url)
+        if ext:
+            score = self._exact_types.get(ext, None)
+            if score is not None:
+                return score
+                
+        # Slow path: regex patterns
+        for pattern, score in self._regex_types:
+            if pattern.search(url):
+                return score
+
+        return 0.0
+
+class FreshnessScorer(URLScorer):
+    __slots__ = ('_weight', '_date_pattern', '_current_year')
+
+    def __init__(self, weight: float = 1.0, current_year: int = 2024):
+        """Initialize freshness scorer.
+        
+        Extracts and scores dates from URLs using format:
+        - YYYY/MM/DD 
+        - YYYY-MM-DD
+        - YYYY_MM_DD
+        - YYYY (year only)
+        
+        Args:
+            weight: Score multiplier
+            current_year: Year to calculate freshness against (default 2024)
+        """
+        super().__init__(weight=weight)
+        self._current_year = current_year
+        
+        # Combined pattern for all date formats
+        # Uses non-capturing groups (?:) and alternation
+        self._date_pattern = re.compile(
+            r'(?:/'  # Path separator
+            r'|[-_])'  # or date separators
+            r'((?:19|20)\d{2})'  # Year group (1900-2099)
+            r'(?:'  # Optional month/day group
+            r'(?:/|[-_])'  # Date separator  
+            r'(?:\d{2})'  # Month
+            r'(?:'  # Optional day
+            r'(?:/|[-_])'  # Date separator
+            r'(?:\d{2})'  # Day
+            r')?'  # Day is optional
+            r')?'  # Month/day group is optional
+        )
+
+    @lru_cache(maxsize=10000)
+    def _extract_year(self, url: str) -> Optional[int]:
+        """Extract the most recent year from URL.
+        
+        Args:
+            url: URL to extract year from
+            
+        Returns:
+            Year as int or None if no valid year found
+        """
+        matches = self._date_pattern.finditer(url)
+        latest_year = None
+        
+        # Find most recent year
+        for match in matches:
+            year = int(match.group(1))
+            if (year <= self._current_year and  # Sanity check
+                (latest_year is None or year > latest_year)):
+                latest_year = year
+                
+        return latest_year
+
+    @lru_cache(maxsize=10000) 
+    def _calculate_score(self, url: str) -> float:
+        """Calculate freshness score based on URL date.
+        
+        More recent years score higher. Uses pre-computed scoring
+        table for common year differences.
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Score between 0.0 and 1.0 * weight
+        """
+        year = self._extract_year(url)
+        if year is None:
+            return 0.5  # Default score
+            
+        # Use lookup table for common year differences
+        year_diff = self._current_year - year
+        if year_diff < len(_FRESHNESS_SCORES):
+            return _FRESHNESS_SCORES[year_diff]
+            
+        # Fallback calculation for older content
+        return max(0.1, 1.0 - year_diff * 0.1)
+
+class DomainAuthorityScorer(URLScorer):
+    __slots__ = ('_weight', '_domain_weights', '_default_weight', '_top_domains')
+    
+    def __init__(
+        self,
+        domain_weights: Dict[str, float],
+        default_weight: float = 0.5,
+        weight: float = 1.0,
+    ):
+        """Initialize domain authority scorer.
+        
+        Args:
+            domain_weights: Dict mapping domains to authority scores
+            default_weight: Score for unknown domains
+            weight: Overall scorer weight multiplier
+            
+        Example:
+            {
+                'python.org': 1.0,
+                'github.com': 0.9,
+                'medium.com': 0.7
+            }
+        """
+        super().__init__(weight=weight)
+        
+        # Pre-process domains for faster lookup
+        self._domain_weights = {
+            domain.lower(): score 
+            for domain, score in domain_weights.items()
+        }
+        self._default_weight = default_weight
+        
+        # Cache top domains for fast path
+        self._top_domains = {
+            domain: score
+            for domain, score in sorted(
+                domain_weights.items(), 
+                key=lambda x: -x[1]
+            )[:5]  # Keep top 5 highest scoring domains
+        }
+
+    @staticmethod
+    @lru_cache(maxsize=10000)
+    def _extract_domain(url: str) -> str:
+        """Extract domain from URL ultra-fast.
+        
+        Handles:
+        - Basic domains: "example.com"
+        - Subdomains: "sub.example.com" 
+        - Ports: "example.com:8080"
+        - IPv4: "192.168.1.1"
+        
+        Args:
+            url: Full URL to extract domain from
+            
+        Returns:
+            Lowercase domain without port
+        """
+        # Find domain start
+        start = url.find('://') 
+        if start == -1:
+            start = 0
+        else:
+            start += 3
+            
+        # Find domain end
+        end = url.find('/', start)
+        if end == -1:
+            end = url.find('?', start)
+            if end == -1:
+                end = url.find('#', start)
+                if end == -1:
+                    end = len(url)
+                    
+        # Extract domain and remove port
+        domain = url[start:end]
+        port_idx = domain.rfind(':')
+        if port_idx != -1:
+            domain = domain[:port_idx]
+            
+        return domain.lower()
+
+    @lru_cache(maxsize=10000)
+    def _calculate_score(self, url: str) -> float:
+        """Calculate domain authority score.
+        
+        Uses staged approach:
+        1. Check top domains (fastest)
+        2. Check full domain weights
+        3. Return default weight
+        
+        Args:
+            url: URL to score
+            
+        Returns:
+            Authority score between 0.0 and 1.0 * weight
+        """
+        domain = self._extract_domain(url)
+        
+        # Fast path: check top domains first
+        score = self._top_domains.get(domain)
+        if score is not None:
+            return score
+            
+        # Regular path: check all domains
+        return self._domain_weights.get(domain, self._default_weight)

crawl4ai/docker_client.py

@@ -0,0 +1,170 @@
+from typing import List, Optional, Union, AsyncGenerator, Dict, Any
+import httpx
+import json
+from urllib.parse import urljoin
+import asyncio
+
+from .async_configs import BrowserConfig, CrawlerRunConfig
+from .models import CrawlResult
+from .async_logger import AsyncLogger, LogLevel
+
+
+class Crawl4aiClientError(Exception):
+    """Base exception for Crawl4ai Docker client errors."""
+    pass
+
+
+class ConnectionError(Crawl4aiClientError):
+    """Raised when connection to the Docker server fails."""
+    pass
+
+
+class RequestError(Crawl4aiClientError):
+    """Raised when the server returns an error response."""
+    pass
+
+
+class Crawl4aiDockerClient:
+    """Client for interacting with Crawl4AI Docker server with token authentication."""
+    
+    def __init__(
+        self,
+        base_url: str = "http://localhost:8000",
+        timeout: float = 30.0,
+        verify_ssl: bool = True,
+        verbose: bool = True,
+        log_file: Optional[str] = None
+    ):
+        self.base_url = base_url.rstrip('/')
+        self.timeout = timeout
+        self.logger = AsyncLogger(log_file=log_file, log_level=LogLevel.DEBUG, verbose=verbose)
+        self._http_client = httpx.AsyncClient(
+            timeout=timeout,
+            verify=verify_ssl,
+            headers={"Content-Type": "application/json"}
+        )
+        self._token: Optional[str] = None
+
+    async def authenticate(self, email: str) -> None:
+        """Authenticate with the server and store the token."""
+        url = urljoin(self.base_url, "/token")
+        try:
+            self.logger.info(f"Authenticating with email: {email}", tag="AUTH")
+            response = await self._http_client.post(url, json={"email": email})
+            response.raise_for_status()
+            data = response.json()
+            self._token = data["access_token"]
+            self._http_client.headers["Authorization"] = f"Bearer {self._token}"
+            self.logger.success("Authentication successful", tag="AUTH")
+        except (httpx.RequestError, httpx.HTTPStatusError) as e:
+            error_msg = f"Authentication failed: {str(e)}"
+            self.logger.error(error_msg, tag="ERROR")
+            raise ConnectionError(error_msg)
+
+    async def _check_server(self) -> None:
+        """Check if server is reachable, raising an error if not."""
+        try:
+            await self._http_client.get(urljoin(self.base_url, "/health"))
+            self.logger.success(f"Connected to {self.base_url}", tag="READY")
+        except httpx.RequestError as e:
+            self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR")
+            raise ConnectionError(f"Cannot connect to server: {str(e)}")
+
+    def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, 
+                       crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]:
+        """Prepare request data from configs."""
+        return {
+            "urls": urls,
+            "browser_config": browser_config.dump() if browser_config else {},
+            "crawler_config": crawler_config.dump() if crawler_config else {}
+        }
+
+    async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
+        """Make an HTTP request with error handling."""
+        url = urljoin(self.base_url, endpoint)
+        try:
+            response = await self._http_client.request(method, url, **kwargs)
+            response.raise_for_status()
+            return response
+        except httpx.TimeoutException as e:
+            raise ConnectionError(f"Request timed out: {str(e)}")
+        except httpx.RequestError as e:
+            raise ConnectionError(f"Failed to connect: {str(e)}")
+        except httpx.HTTPStatusError as e:
+            error_msg = (e.response.json().get("detail", str(e)) 
+                        if "application/json" in e.response.headers.get("content-type", "") 
+                        else str(e))
+            raise RequestError(f"Server error {e.response.status_code}: {error_msg}")
+
+    async def crawl(
+        self,
+        urls: List[str],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None
+    ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
+        """Execute a crawl operation."""
+        if not self._token:
+            raise Crawl4aiClientError("Authentication required. Call authenticate() first.")
+        await self._check_server()
+        
+        data = self._prepare_request(urls, browser_config, crawler_config)
+        is_streaming = crawler_config and crawler_config.stream
+        
+        self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL")
+        
+        if is_streaming:
+            async def stream_results() -> AsyncGenerator[CrawlResult, None]:
+                async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response:
+                    response.raise_for_status()
+                    async for line in response.aiter_lines():
+                        if line.strip():
+                            result = json.loads(line)
+                            if "error" in result:
+                                self.logger.error_status(url=result.get("url", "unknown"), error=result["error"])
+                                continue
+                            self.logger.url_status(url=result.get("url", "unknown"), success=True, timing=result.get("timing", 0.0))
+                            if result.get("status") == "completed":
+                                continue
+                            else:
+                                yield CrawlResult(**result)
+            return stream_results()
+        
+        response = await self._request("POST", "/crawl", json=data)
+        result_data = response.json()
+        if not result_data.get("success", False):
+            raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}")
+        
+        results = [CrawlResult(**r) for r in result_data.get("results", [])]
+        self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL")
+        return results[0] if len(results) == 1 else results
+
+    async def get_schema(self) -> Dict[str, Any]:
+        """Retrieve configuration schemas."""
+        if not self._token:
+            raise Crawl4aiClientError("Authentication required. Call authenticate() first.")
+        response = await self._request("GET", "/schema")
+        return response.json()
+
+    async def close(self) -> None:
+        """Close the HTTP client session."""
+        self.logger.info("Closing client", tag="CLOSE")
+        await self._http_client.aclose()
+
+    async def __aenter__(self) -> "Crawl4aiDockerClient":
+        return self
+
+    async def __aexit__(self, exc_type: Optional[type], exc_val: Optional[Exception], exc_tb: Optional[Any]) -> None:
+        await self.close()
+
+
+# Example usage
+async def main():
+    async with Crawl4aiDockerClient(verbose=True) as client:
+        await client.authenticate("user@example.com")
+        result = await client.crawl(["https://example.com"])
+        print(result)
+        schema = await client.get_schema()
+        print(schema)
+
+if __name__ == "__main__":
+    asyncio.run(main())

crawl4ai/html2text/__init__.py

@@ -54,13 +54,13 @@ class HTML2Text(html.parser.HTMLParser):
         self.td_count = 0
         self.table_start = False
         self.unicode_snob = config.UNICODE_SNOB  # covered in cli
-        
+
         self.escape_snob = config.ESCAPE_SNOB  # covered in cli
         self.escape_backslash = config.ESCAPE_BACKSLASH  # covered in cli
         self.escape_dot = config.ESCAPE_DOT  # covered in cli
         self.escape_plus = config.ESCAPE_PLUS  # covered in cli
         self.escape_dash = config.ESCAPE_DASH  # covered in cli
-        
+
         self.links_each_paragraph = config.LINKS_EACH_PARAGRAPH
         self.body_width = bodywidth  # covered in cli
         self.skip_internal_links = config.SKIP_INTERNAL_LINKS  # covered in cli
@@ -144,8 +144,8 @@ class HTML2Text(html.parser.HTMLParser):
 
     def update_params(self, **kwargs):
         for key, value in kwargs.items():
-            setattr(self, key, value) 
-    
+            setattr(self, key, value)
+
     def feed(self, data: str) -> None:
         data = data.replace("</' + 'script>", "</ignore>")
         super().feed(data)
@@ -510,6 +510,7 @@ class HTML2Text(html.parser.HTMLParser):
 
         if tag == "a" and not self.ignore_links:
             if start:
+                self.inside_link = True
                 if (
                     "href" in attrs
                     and attrs["href"] is not None
@@ -526,6 +527,7 @@ class HTML2Text(html.parser.HTMLParser):
                 else:
                     self.astack.append(None)
             else:
+                self.inside_link = False
                 if self.astack:
                     a = self.astack.pop()
                     if self.maybe_automatic_link and not self.empty_link:
@@ -610,13 +612,22 @@ class HTML2Text(html.parser.HTMLParser):
                         self.o("[" + str(a_props.count) + "]")
 
         if tag == "dl" and start:
-            self.p()
-        if tag == "dt" and not start:
-            self.pbr()
-        if tag == "dd" and start:
-            self.o("    ")
-        if tag == "dd" and not start:
-            self.pbr()
+            self.p()  # Add paragraph break before list starts
+            self.p_p = 0  # Reset paragraph state
+        
+        elif tag == "dt" and start:
+            if self.p_p == 0:  # If not first term
+                self.o("\n\n")  # Add spacing before new term-definition pair
+            self.p_p = 0  # Reset paragraph state
+        
+        elif tag == "dt" and not start:
+            self.o("\n")  # Single newline between term and definition
+        
+        elif tag == "dd" and start:
+            self.o("    ")  # Indent definition
+        
+        elif tag == "dd" and not start:
+            self.p_p = 0
 
         if tag in ["ol", "ul"]:
             # Google Docs create sub lists as top level lists
@@ -903,7 +914,13 @@ class HTML2Text(html.parser.HTMLParser):
                 self.empty_link = False
 
         if not self.code and not self.pre and not entity_char:
-            data = escape_md_section(data, snob=self.escape_snob, escape_dot=self.escape_dot, escape_plus=self.escape_plus, escape_dash=self.escape_dash)
+            data = escape_md_section(
+                data,
+                snob=self.escape_snob,
+                escape_dot=self.escape_dot,
+                escape_plus=self.escape_plus,
+                escape_dash=self.escape_dash,
+            )
         self.preceding_data = data
         self.o(data, puredata=True)
 
@@ -1013,3 +1030,141 @@ def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = None) ->
     h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
 
     return h.handle(html)
+
+
+class CustomHTML2Text(HTML2Text):
+    def __init__(self, *args, handle_code_in_pre=False, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.inside_pre = False
+        self.inside_code = False
+        self.inside_link = False
+        self.preserve_tags = set()  # Set of tags to preserve
+        self.current_preserved_tag = None
+        self.preserved_content = []
+        self.preserve_depth = 0
+        self.handle_code_in_pre = handle_code_in_pre
+
+        # Configuration options
+        self.skip_internal_links = False
+        self.single_line_break = False
+        self.mark_code = False
+        self.include_sup_sub = False
+        self.body_width = 0
+        self.ignore_mailto_links = True
+        self.ignore_links = False
+        self.escape_backslash = False
+        self.escape_dot = False
+        self.escape_plus = False
+        self.escape_dash = False
+        self.escape_snob = False
+
+    def update_params(self, **kwargs):
+        """Update parameters and set preserved tags."""
+        for key, value in kwargs.items():
+            if key == "preserve_tags":
+                self.preserve_tags = set(value)
+            elif key == "handle_code_in_pre":
+                self.handle_code_in_pre = value
+            else:
+                setattr(self, key, value)
+
+    def handle_tag(self, tag, attrs, start):
+        # Handle preserved tags
+        if tag in self.preserve_tags:
+            if start:
+                if self.preserve_depth == 0:
+                    self.current_preserved_tag = tag
+                    self.preserved_content = []
+                    # Format opening tag with attributes
+                    attr_str = "".join(
+                        f' {k}="{v}"' for k, v in attrs.items() if v is not None
+                    )
+                    self.preserved_content.append(f"<{tag}{attr_str}>")
+                self.preserve_depth += 1
+                return
+            else:
+                self.preserve_depth -= 1
+                if self.preserve_depth == 0:
+                    self.preserved_content.append(f"</{tag}>")
+                    # Output the preserved HTML block with proper spacing
+                    preserved_html = "".join(self.preserved_content)
+                    self.o("\n" + preserved_html + "\n")
+                    self.current_preserved_tag = None
+                return
+
+        # If we're inside a preserved tag, collect all content
+        if self.preserve_depth > 0:
+            if start:
+                # Format nested tags with attributes
+                attr_str = "".join(
+                    f' {k}="{v}"' for k, v in attrs.items() if v is not None
+                )
+                self.preserved_content.append(f"<{tag}{attr_str}>")
+            else:
+                self.preserved_content.append(f"</{tag}>")
+            return
+
+        # Handle pre tags
+        if tag == "pre":
+            if start:
+                self.o("```\n")  # Markdown code block start
+                self.inside_pre = True
+            else:
+                self.o("\n```\n")  # Markdown code block end
+                self.inside_pre = False
+        elif tag == "code":
+            if self.inside_pre and not self.handle_code_in_pre:
+                # Ignore code tags inside pre blocks if handle_code_in_pre is False
+                return
+            if start:
+                if not self.inside_link:
+                    self.o("`")  # Only output backtick if not inside a link
+                self.inside_code = True
+            else:
+                if not self.inside_link:
+                    self.o("`")  # Only output backtick if not inside a link
+                self.inside_code = False
+
+            # If inside a link, let the parent class handle the content
+            if self.inside_link:
+                super().handle_tag(tag, attrs, start) 
+        else:
+            super().handle_tag(tag, attrs, start)
+
+    def handle_data(self, data, entity_char=False):
+        """Override handle_data to capture content within preserved tags."""
+        if self.preserve_depth > 0:
+            self.preserved_content.append(data)
+            return
+
+        if self.inside_pre:
+            # Output the raw content for pre blocks, including content inside code tags
+            self.o(data)  # Directly output the data as-is (preserve newlines)
+            return
+        if self.inside_code:
+            # Inline code: no newlines allowed
+            self.o(data.replace("\n", " "))
+            return
+
+        # Default behavior for other tags
+        super().handle_data(data, entity_char)
+
+    #     # Handle pre tags
+    #     if tag == 'pre':
+    #         if start:
+    #             self.o('```\n')
+    #             self.inside_pre = True
+    #         else:
+    #             self.o('\n```')
+    #             self.inside_pre = False
+    #     # elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+    #     #     pass
+    #     else:
+    #         super().handle_tag(tag, attrs, start)
+
+    # def handle_data(self, data, entity_char=False):
+    #     """Override handle_data to capture content within preserved tags."""
+    #     if self.preserve_depth > 0:
+    #         self.preserved_content.append(data)
+    #         return
+    #     super().handle_data(data, entity_char)

crawl4ai/html2text/_typing.py

@@ -1,2 +1,3 @@
 class OutCallback:
-    def __call__(self, s: str) -> None: ...
+    def __call__(self, s: str) -> None:
+        ...

crawl4ai/html2text/utils.py

@@ -210,7 +210,7 @@ def escape_md_section(
     snob: bool = False,
     escape_dot: bool = True,
     escape_plus: bool = True,
-    escape_dash: bool = True
+    escape_dash: bool = True,
 ) -> str:
     """
     Escapes markdown-sensitive characters across whole document sections.
@@ -233,6 +233,7 @@ def escape_md_section(
 
     return text
 
+
 def reformat_table(lines: List[str], right_margin: int) -> List[str]:
     """
     Given the lines of a table

crawl4ai/hub.py

@@ -0,0 +1,69 @@
+# crawl4ai/hub.py
+from abc import ABC, abstractmethod
+from typing import Dict, Type, Union
+import logging
+import importlib
+from pathlib import Path
+import inspect
+
+logger = logging.getLogger(__name__)
+
+
+class BaseCrawler(ABC):
+    def __init__(self):
+        self.logger = logging.getLogger(self.__class__.__name__)
+        
+    @abstractmethod
+    async def run(self, url: str = "", **kwargs) -> str:
+        """
+        Implement this method to return JSON string.
+        Must accept URL + arbitrary kwargs for flexibility.
+        """
+        pass
+
+    def __init_subclass__(cls, **kwargs):
+        """Enforce interface validation on subclassing"""
+        super().__init_subclass__(**kwargs)
+        
+        # Verify run method signature
+        run_method = cls.run
+        if not run_method.__code__.co_argcount >= 2:  # self + url
+            raise TypeError(f"{cls.__name__} must implement 'run(self, url: str, **kwargs)'")
+            
+        # Verify async nature
+        if not inspect.iscoroutinefunction(run_method):
+            raise TypeError(f"{cls.__name__}.run must be async")
+
+class CrawlerHub:
+    _crawlers: Dict[str, Type[BaseCrawler]] = {}
+
+    @classmethod
+    def _discover_crawlers(cls):
+        """Dynamically load crawlers from /crawlers in 3 lines"""
+        base_path = Path(__file__).parent / "crawlers"
+        for crawler_dir in base_path.iterdir():
+            if crawler_dir.is_dir():
+                try:
+                    module = importlib.import_module(
+                        f"crawl4ai.crawlers.{crawler_dir.name}.crawler"
+                    )
+                    for attr in dir(module):
+                        cls._maybe_register_crawler(
+                            getattr(module, attr), crawler_dir.name
+                        )
+                except Exception as e:
+                    logger.warning(f"Failed {crawler_dir.name}: {str(e)}")
+
+    @classmethod
+    def _maybe_register_crawler(cls, obj, name: str):
+        """Brilliant one-liner registration"""
+        if isinstance(obj, type) and issubclass(obj, BaseCrawler) and obj != BaseCrawler:
+            module = importlib.import_module(obj.__module__)
+            obj.meta = getattr(module, "__meta__", {})
+            cls._crawlers[name] = obj
+
+    @classmethod
+    def get(cls, name: str) -> Union[Type[BaseCrawler], None]:
+        if not cls._crawlers:
+            cls._discover_crawlers()
+        return cls._crawlers.get(name)

crawl4ai/install.py

@@ -0,0 +1,143 @@
+import subprocess
+import sys
+import asyncio
+from .async_logger import AsyncLogger, LogLevel
+from pathlib import Path
+import os
+import shutil
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+def setup_home_directory():
+    """Set up the .crawl4ai folder structure in the user's home directory."""
+    base_dir = os.getenv("CRAWL4_AI_BASE_DIRECTORY")
+    crawl4ai_folder = Path(base_dir) if base_dir else Path.home()
+    crawl4ai_config = crawl4ai_folder / "global.yml"
+    crawl4ai_folder = crawl4ai_folder / ".crawl4ai"
+    cache_folder = crawl4ai_folder / "cache"
+    content_folders = [
+        "html_content",
+        "cleaned_html",
+        "markdown_content",
+        "extracted_content",
+        "screenshots",
+    ]
+
+    # Clean up old cache if exists
+    if cache_folder.exists():
+        shutil.rmtree(cache_folder)
+
+    # Create new folder structure
+    crawl4ai_folder.mkdir(exist_ok=True)
+    cache_folder.mkdir(exist_ok=True)
+    for folder in content_folders:
+        (crawl4ai_folder / folder).mkdir(exist_ok=True)
+    
+    # If config file does not exist, create it
+    if not crawl4ai_config.exists():
+        with open(crawl4ai_config, "w") as f:
+            f.write("")
+
+def post_install():
+    """Run all post-installation tasks"""
+    logger.info("Running post-installation setup...", tag="INIT")
+    setup_home_directory()
+    install_playwright()
+    run_migration()
+    logger.success("Post-installation setup completed!", tag="COMPLETE")
+
+
+def install_playwright():
+    logger.info("Installing Playwright browsers...", tag="INIT")
+    try:
+        # subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chrome"])
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                "playwright",
+                "install",
+                "--with-deps",
+                "--force",
+                "chromium",
+            ]
+        )
+        logger.success(
+            "Playwright installation completed successfully.", tag="COMPLETE"
+        )
+    except subprocess.CalledProcessError:
+        # logger.error(f"Error during Playwright installation: {e}", tag="ERROR")
+        logger.warning(
+            f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
+        )
+    except Exception:
+        # logger.error(f"Unexpected error during Playwright installation: {e}", tag="ERROR")
+        logger.warning(
+            f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
+        )
+
+
+def run_migration():
+    """Initialize database during installation"""
+    try:
+        logger.info("Starting database initialization...", tag="INIT")
+        from crawl4ai.async_database import async_db_manager
+
+        asyncio.run(async_db_manager.initialize())
+        logger.success(
+            "Database initialization completed successfully.", tag="COMPLETE"
+        )
+    except ImportError:
+        logger.warning("Database module not found. Will initialize on first use.")
+    except Exception as e:
+        logger.warning(f"Database initialization failed: {e}")
+        logger.warning("Database will be initialized on first use")
+
+
+async def run_doctor():
+    """Test if Crawl4AI is working properly"""
+    logger.info("Running Crawl4AI health check...", tag="INIT")
+    try:
+        from .async_webcrawler import (
+            AsyncWebCrawler,
+            BrowserConfig,
+            CrawlerRunConfig,
+            CacheMode,
+        )
+
+        browser_config = BrowserConfig(
+            headless=True,
+            browser_type="chromium",
+            ignore_https_errors=True,
+            light_mode=True,
+            viewport_width=1280,
+            viewport_height=720,
+        )
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            screenshot=True,
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            logger.info("Testing crawling capabilities...", tag="TEST")
+            result = await crawler.arun(url="https://crawl4ai.com", config=run_config)
+
+            if result and result.markdown:
+                logger.success("✅ Crawling test passed!", tag="COMPLETE")
+                return True
+            else:
+                raise Exception("Failed to get content")
+
+    except Exception as e:
+        logger.error(f"❌ Test failed: {e}", tag="ERROR")
+        return False
+
+
+def doctor():
+    """Entry point for the doctor command"""
+    import asyncio
+
+    asyncio.run(run_doctor())
+    sys.exit(0)

crawl4ai/js_snippet/__init__.py

@@ -0,0 +1,18 @@
+import os
+
+
+# Create a function get name of a js script, then load from the CURRENT folder of this script and return its content as string, make sure its error free
+def load_js_script(script_name):
+    # Get the path of the current script
+    current_script_path = os.path.dirname(os.path.realpath(__file__))
+    # Get the path of the script to load
+    script_path = os.path.join(current_script_path, script_name + ".js")
+    # Check if the script exists
+    if not os.path.exists(script_path):
+        raise ValueError(
+            f"Script {script_name} not found in the folder {current_script_path}"
+        )
+    # Load the content of the script
+    with open(script_path, "r") as f:
+        script_content = f.read()
+    return script_content

crawl4ai/js_snippet/navigator_overrider.js

@@ -0,0 +1,25 @@
+// Pass the Permissions Test.
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) =>
+    parameters.name === "notifications"
+        ? Promise.resolve({ state: Notification.permission })
+        : originalQuery(parameters);
+Object.defineProperty(navigator, "webdriver", {
+    get: () => undefined,
+});
+window.navigator.chrome = {
+    runtime: {},
+    // Add other properties if necessary
+};
+Object.defineProperty(navigator, "plugins", {
+    get: () => [1, 2, 3, 4, 5],
+});
+Object.defineProperty(navigator, "languages", {
+    get: () => ["en-US", "en"],
+});
+Object.defineProperty(document, "hidden", {
+    get: () => false,
+});
+Object.defineProperty(document, "visibilityState", {
+    get: () => "visible",
+});

crawl4ai/js_snippet/remove_overlay_elements.js

@@ -0,0 +1,119 @@
+async () => {
+    // Function to check if element is visible
+    const isVisible = (elem) => {
+        const style = window.getComputedStyle(elem);
+        return style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0";
+    };
+
+    // Common selectors for popups and overlays
+    const commonSelectors = [
+        // Close buttons first
+        'button[class*="close" i]',
+        'button[class*="dismiss" i]',
+        'button[aria-label*="close" i]',
+        'button[title*="close" i]',
+        'a[class*="close" i]',
+        'span[class*="close" i]',
+
+        // Cookie notices
+        '[class*="cookie-banner" i]',
+        '[id*="cookie-banner" i]',
+        '[class*="cookie-consent" i]',
+        '[id*="cookie-consent" i]',
+
+        // Newsletter/subscription dialogs
+        '[class*="newsletter" i]',
+        '[class*="subscribe" i]',
+
+        // Generic popups/modals
+        '[class*="popup" i]',
+        '[class*="modal" i]',
+        '[class*="overlay" i]',
+        '[class*="dialog" i]',
+        '[role="dialog"]',
+        '[role="alertdialog"]',
+    ];
+
+    // Try to click close buttons first
+    for (const selector of commonSelectors.slice(0, 6)) {
+        const closeButtons = document.querySelectorAll(selector);
+        for (const button of closeButtons) {
+            if (isVisible(button)) {
+                try {
+                    button.click();
+                    await new Promise((resolve) => setTimeout(resolve, 100));
+                } catch (e) {
+                    console.log("Error clicking button:", e);
+                }
+            }
+        }
+    }
+
+    // Remove remaining overlay elements
+    const removeOverlays = () => {
+        // Find elements with high z-index
+        const allElements = document.querySelectorAll("*");
+        for (const elem of allElements) {
+            const style = window.getComputedStyle(elem);
+            const zIndex = parseInt(style.zIndex);
+            const position = style.position;
+
+            if (
+                isVisible(elem) &&
+                (zIndex > 999 || position === "fixed" || position === "absolute") &&
+                (elem.offsetWidth > window.innerWidth * 0.5 ||
+                    elem.offsetHeight > window.innerHeight * 0.5 ||
+                    style.backgroundColor.includes("rgba") ||
+                    parseFloat(style.opacity) < 1)
+            ) {
+                elem.remove();
+            }
+        }
+
+        // Remove elements matching common selectors
+        for (const selector of commonSelectors) {
+            const elements = document.querySelectorAll(selector);
+            elements.forEach((elem) => {
+                if (isVisible(elem)) {
+                    elem.remove();
+                }
+            });
+        }
+    };
+
+    // Remove overlay elements
+    removeOverlays();
+
+    // Remove any fixed/sticky position elements at the top/bottom
+    const removeFixedElements = () => {
+        const elements = document.querySelectorAll("*");
+        elements.forEach((elem) => {
+            const style = window.getComputedStyle(elem);
+            if ((style.position === "fixed" || style.position === "sticky") && isVisible(elem)) {
+                elem.remove();
+            }
+        });
+    };
+
+    removeFixedElements();
+
+    // Remove empty block elements as: div, p, span, etc.
+    const removeEmptyBlockElements = () => {
+        const blockElements = document.querySelectorAll(
+            "div, p, span, section, article, header, footer, aside, nav, main, ul, ol, li, dl, dt, dd, h1, h2, h3, h4, h5, h6"
+        );
+        blockElements.forEach((elem) => {
+            if (elem.innerText.trim() === "") {
+                elem.remove();
+            }
+        });
+    };
+
+    // Remove margin-right and padding-right from body (often added by modal scripts)
+    document.body.style.marginRight = "0px";
+    document.body.style.paddingRight = "0px";
+    document.body.style.overflow = "auto";
+
+    // Wait a bit for any animations to complete
+    await new Promise((resolve) => setTimeout(resolve, 100));
+};

crawl4ai/js_snippet/update_image_dimensions.js

@@ -0,0 +1,54 @@
+() => {
+    return new Promise((resolve) => {
+        const filterImage = (img) => {
+            // Filter out images that are too small
+            if (img.width < 100 && img.height < 100) return false;
+
+            // Filter out images that are not visible
+            const rect = img.getBoundingClientRect();
+            if (rect.width === 0 || rect.height === 0) return false;
+
+            // Filter out images with certain class names (e.g., icons, thumbnails)
+            if (img.classList.contains("icon") || img.classList.contains("thumbnail")) return false;
+
+            // Filter out images with certain patterns in their src (e.g., placeholder images)
+            if (img.src.includes("placeholder") || img.src.includes("icon")) return false;
+
+            return true;
+        };
+
+        const images = Array.from(document.querySelectorAll("img")).filter(filterImage);
+        let imagesLeft = images.length;
+
+        if (imagesLeft === 0) {
+            resolve();
+            return;
+        }
+
+        const checkImage = (img) => {
+            if (img.complete && img.naturalWidth !== 0) {
+                img.setAttribute("width", img.naturalWidth);
+                img.setAttribute("height", img.naturalHeight);
+                imagesLeft--;
+                if (imagesLeft === 0) resolve();
+            }
+        };
+
+        images.forEach((img) => {
+            checkImage(img);
+            if (!img.complete) {
+                img.onload = () => {
+                    checkImage(img);
+                };
+                img.onerror = () => {
+                    imagesLeft--;
+                    if (imagesLeft === 0) resolve();
+                };
+            }
+        });
+
+        // Fallback timeout of 5 seconds
+        // setTimeout(() => resolve(), 5000);
+        resolve();
+    });
+};

crawl4ai/legacy/cli.py

@@ -0,0 +1,123 @@
+import click
+import sys
+import asyncio
+from typing import List
+from .docs_manager import DocsManager
+from .async_logger import AsyncLogger
+
+logger = AsyncLogger(verbose=True)
+docs_manager = DocsManager(logger)
+
+
+def print_table(headers: List[str], rows: List[List[str]], padding: int = 2):
+    """Print formatted table with headers and rows"""
+    widths = [max(len(str(cell)) for cell in col) for col in zip(headers, *rows)]
+    border = "+" + "+".join("-" * (w + 2 * padding) for w in widths) + "+"
+
+    def format_row(row):
+        return (
+            "|"
+            + "|".join(
+                f"{' ' * padding}{str(cell):<{w}}{' ' * padding}"
+                for cell, w in zip(row, widths)
+            )
+            + "|"
+        )
+
+    click.echo(border)
+    click.echo(format_row(headers))
+    click.echo(border)
+    for row in rows:
+        click.echo(format_row(row))
+    click.echo(border)
+
+
+@click.group()
+def cli():
+    """Crawl4AI Command Line Interface"""
+    pass
+
+
+@cli.group()
+def docs():
+    """Documentation operations"""
+    pass
+
+
+@docs.command()
+@click.argument("sections", nargs=-1)
+@click.option(
+    "--mode", type=click.Choice(["extended", "condensed"]), default="extended"
+)
+def combine(sections: tuple, mode: str):
+    """Combine documentation sections"""
+    try:
+        asyncio.run(docs_manager.ensure_docs_exist())
+        click.echo(docs_manager.generate(sections, mode))
+    except Exception as e:
+        logger.error(str(e), tag="ERROR")
+        sys.exit(1)
+
+
+@docs.command()
+@click.argument("query")
+@click.option("--top-k", "-k", default=5)
+@click.option("--build-index", is_flag=True, help="Build index if missing")
+def search(query: str, top_k: int, build_index: bool):
+    """Search documentation"""
+    try:
+        result = docs_manager.search(query, top_k)
+        if result == "No search index available. Call build_search_index() first.":
+            if build_index or click.confirm("No search index found. Build it now?"):
+                asyncio.run(docs_manager.llm_text.generate_index_files())
+                result = docs_manager.search(query, top_k)
+        click.echo(result)
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+@docs.command()
+def update():
+    """Update docs from GitHub"""
+    try:
+        asyncio.run(docs_manager.fetch_docs())
+        click.echo("Documentation updated successfully")
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+@docs.command()
+@click.option("--force-facts", is_flag=True, help="Force regenerate fact files")
+@click.option("--clear-cache", is_flag=True, help="Clear BM25 cache")
+def index(force_facts: bool, clear_cache: bool):
+    """Build or rebuild search indexes"""
+    try:
+        asyncio.run(docs_manager.ensure_docs_exist())
+        asyncio.run(
+            docs_manager.llm_text.generate_index_files(
+                force_generate_facts=force_facts, clear_bm25_cache=clear_cache
+            )
+        )
+        click.echo("Search indexes built successfully")
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+# Add docs list command
+@docs.command()
+def list():
+    """List available documentation sections"""
+    try:
+        sections = docs_manager.list()
+        print_table(["Sections"], [[section] for section in sections])
+
+    except Exception as e:
+        click.echo(f"Error: {str(e)}", err=True)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    cli()

crawl4ai/legacy/crawler_strategy.py

@@ -15,54 +15,53 @@ import logging, time
 import base64
 from PIL import Image, ImageDraw, ImageFont
 from io import BytesIO
-from typing import List, Callable
+from typing import Callable
 import requests
 import os
 from pathlib import Path
 from .utils import *
 
-logger = logging.getLogger('selenium.webdriver.remote.remote_connection')
+logger = logging.getLogger("selenium.webdriver.remote.remote_connection")
 logger.setLevel(logging.WARNING)
 
-logger_driver = logging.getLogger('selenium.webdriver.common.service')
+logger_driver = logging.getLogger("selenium.webdriver.common.service")
 logger_driver.setLevel(logging.WARNING)
 
-urllib3_logger = logging.getLogger('urllib3.connectionpool')
+urllib3_logger = logging.getLogger("urllib3.connectionpool")
 urllib3_logger.setLevel(logging.WARNING)
 
 # Disable http.client logging
-http_client_logger = logging.getLogger('http.client')
+http_client_logger = logging.getLogger("http.client")
 http_client_logger.setLevel(logging.WARNING)
 
 # Disable driver_finder and service logging
-driver_finder_logger = logging.getLogger('selenium.webdriver.common.driver_finder')
+driver_finder_logger = logging.getLogger("selenium.webdriver.common.driver_finder")
 driver_finder_logger.setLevel(logging.WARNING)
 
 
-
-
 class CrawlerStrategy(ABC):
     @abstractmethod
     def crawl(self, url: str, **kwargs) -> str:
         pass
-    
+
     @abstractmethod
     def take_screenshot(self, save_path: str):
         pass
-    
+
     @abstractmethod
     def update_user_agent(self, user_agent: str):
         pass
-    
+
     @abstractmethod
     def set_hook(self, hook_type: str, hook: Callable):
         pass
 
+
 class CloudCrawlerStrategy(CrawlerStrategy):
-    def __init__(self, use_cached_html = False):
+    def __init__(self, use_cached_html=False):
         super().__init__()
         self.use_cached_html = use_cached_html
-        
+
     def crawl(self, url: str) -> str:
         data = {
             "urls": [url],
@@ -76,6 +75,7 @@ class CloudCrawlerStrategy(CrawlerStrategy):
         html = response["results"][0]["html"]
         return sanitize_input_encode(html)
 
+
 class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
     def __init__(self, use_cached_html=False, js_code=None, **kwargs):
         super().__init__()
@@ -87,20 +87,25 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         if kwargs.get("user_agent"):
             self.options.add_argument("--user-agent=" + kwargs.get("user_agent"))
         else:
-            user_agent = kwargs.get("user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+            user_agent = kwargs.get(
+                "user_agent",
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+            )
             self.options.add_argument(f"--user-agent={user_agent}")
-            self.options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
-                  
+            self.options.add_argument(
+                "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+            )
+
         self.options.headless = kwargs.get("headless", True)
         if self.options.headless:
             self.options.add_argument("--headless")
-        
-        self.options.add_argument("--disable-gpu")  
+
+        self.options.add_argument("--disable-gpu")
         self.options.add_argument("--window-size=1920,1080")
         self.options.add_argument("--no-sandbox")
         self.options.add_argument("--disable-dev-shm-usage")
-        self.options.add_argument("--disable-blink-features=AutomationControlled")     
-        
+        self.options.add_argument("--disable-blink-features=AutomationControlled")
+
         # self.options.add_argument("--disable-dev-shm-usage")
         self.options.add_argument("--disable-gpu")
         # self.options.add_argument("--disable-extensions")
@@ -120,14 +125,14 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         self.use_cached_html = use_cached_html
         self.js_code = js_code
         self.verbose = kwargs.get("verbose", False)
-        
+
         # Hooks
         self.hooks = {
-            'on_driver_created': None,
-            'on_user_agent_updated': None,
-            'before_get_url': None,
-            'after_get_url': None,
-            'before_return_html': None
+            "on_driver_created": None,
+            "on_user_agent_updated": None,
+            "before_get_url": None,
+            "after_get_url": None,
+            "before_return_html": None,
         }
 
         # chromedriver_autoinstaller.install()
@@ -137,31 +142,28 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         # chromedriver_path = chromedriver_autoinstaller.install()
         # chromedriver_path = chromedriver_autoinstaller.utils.download_chromedriver()
         # self.service = Service(chromedriver_autoinstaller.install())
-        
-        
+
         # chromedriver_path = ChromeDriverManager().install()
         # self.service = Service(chromedriver_path)
         # self.service.log_path = "NUL"
         # self.driver = webdriver.Chrome(service=self.service, options=self.options)
-        
+
         # Use selenium-manager (built into Selenium 4.10.0+)
         self.service = Service()
         self.driver = webdriver.Chrome(options=self.options)
-        
-        self.driver = self.execute_hook('on_driver_created', self.driver)
-        
+
+        self.driver = self.execute_hook("on_driver_created", self.driver)
+
         if kwargs.get("cookies"):
             for cookie in kwargs.get("cookies"):
                 self.driver.add_cookie(cookie)
-            
-        
 
     def set_hook(self, hook_type: str, hook: Callable):
         if hook_type in self.hooks:
             self.hooks[hook_type] = hook
         else:
             raise ValueError(f"Invalid hook type: {hook_type}")
-    
+
     def execute_hook(self, hook_type: str, *args):
         hook = self.hooks.get(hook_type)
         if hook:
@@ -170,7 +172,9 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
                 if isinstance(result, webdriver.Chrome):
                     return result
                 else:
-                    raise TypeError(f"Hook {hook_type} must return an instance of webdriver.Chrome or None.")
+                    raise TypeError(
+                        f"Hook {hook_type} must return an instance of webdriver.Chrome or None."
+                    )
         # If the hook returns None or there is no hook, return self.driver
         return self.driver
 
@@ -178,60 +182,77 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         self.options.add_argument(f"user-agent={user_agent}")
         self.driver.quit()
         self.driver = webdriver.Chrome(service=self.service, options=self.options)
-        self.driver = self.execute_hook('on_user_agent_updated', self.driver)
+        self.driver = self.execute_hook("on_user_agent_updated", self.driver)
 
     def set_custom_headers(self, headers: dict):
         # Enable Network domain for sending headers
-        self.driver.execute_cdp_cmd('Network.enable', {})
+        self.driver.execute_cdp_cmd("Network.enable", {})
         # Set extra HTTP headers
-        self.driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', {'headers': headers})
+        self.driver.execute_cdp_cmd("Network.setExtraHTTPHeaders", {"headers": headers})
 
-    def _ensure_page_load(self,  max_checks=6, check_interval=0.01):
+    def _ensure_page_load(self, max_checks=6, check_interval=0.01):
         initial_length = len(self.driver.page_source)
-        
+
         for ix in range(max_checks):
             # print(f"Checking page load: {ix}")
             time.sleep(check_interval)
             current_length = len(self.driver.page_source)
-            
+
             if current_length != initial_length:
                 break
 
         return self.driver.page_source
-    
+
     def crawl(self, url: str, **kwargs) -> str:
         # Create md5 hash of the URL
         import hashlib
+
         url_hash = hashlib.md5(url.encode()).hexdigest()
-        
+
         if self.use_cached_html:
-            cache_file_path = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai", "cache", url_hash)
+            cache_file_path = os.path.join(
+                os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()),
+                ".crawl4ai",
+                "cache",
+                url_hash,
+            )
             if os.path.exists(cache_file_path):
                 with open(cache_file_path, "r") as f:
                     return sanitize_input_encode(f.read())
 
         try:
-            self.driver = self.execute_hook('before_get_url', self.driver)
+            self.driver = self.execute_hook("before_get_url", self.driver)
             if self.verbose:
                 print(f"[LOG] 🕸️ Crawling {url} using LocalSeleniumCrawlerStrategy...")
-            self.driver.get(url) #<html><head></head><body></body></html>
-            
+            self.driver.get(url)  # <html><head></head><body></body></html>
+
             WebDriverWait(self.driver, 20).until(
-                lambda d: d.execute_script('return document.readyState') == 'complete'
+                lambda d: d.execute_script("return document.readyState") == "complete"
             )
             WebDriverWait(self.driver, 10).until(
                 EC.presence_of_all_elements_located((By.TAG_NAME, "body"))
             )
-            
-            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
-            
-            self.driver = self.execute_hook('after_get_url', self.driver)
-            html = sanitize_input_encode(self._ensure_page_load()) # self.driver.page_source                                        
-            can_not_be_done_headless = False # Look at my creativity for naming variables
-            
+
+            self.driver.execute_script(
+                "window.scrollTo(0, document.body.scrollHeight);"
+            )
+
+            self.driver = self.execute_hook("after_get_url", self.driver)
+            html = sanitize_input_encode(
+                self._ensure_page_load()
+            )  # self.driver.page_source
+            can_not_be_done_headless = (
+                False  # Look at my creativity for naming variables
+            )
+
             # TODO: Very ugly approach, but promise to change it!
-            if kwargs.get('bypass_headless', False) or html == "<html><head></head><body></body></html>":
-                print("[LOG] 🙌 Page could not be loaded in headless mode. Trying non-headless mode...")
+            if (
+                kwargs.get("bypass_headless", False)
+                or html == "<html><head></head><body></body></html>"
+            ):
+                print(
+                    "[LOG] 🙌 Page could not be loaded in headless mode. Trying non-headless mode..."
+                )
                 can_not_be_done_headless = True
                 options = Options()
                 options.headless = False
@@ -239,27 +260,31 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
                 options.add_argument("--window-size=5,5")
                 driver = webdriver.Chrome(service=self.service, options=options)
                 driver.get(url)
-                self.driver = self.execute_hook('after_get_url', driver)
+                self.driver = self.execute_hook("after_get_url", driver)
                 html = sanitize_input_encode(driver.page_source)
                 driver.quit()
-            
+
             # Execute JS code if provided
             self.js_code = kwargs.get("js_code", self.js_code)
             if self.js_code and type(self.js_code) == str:
                 self.driver.execute_script(self.js_code)
                 # Optionally, wait for some condition after executing the JS code
                 WebDriverWait(self.driver, 10).until(
-                    lambda driver: driver.execute_script("return document.readyState") == "complete"
+                    lambda driver: driver.execute_script("return document.readyState")
+                    == "complete"
                 )
             elif self.js_code and type(self.js_code) == list:
                 for js in self.js_code:
                     self.driver.execute_script(js)
                     WebDriverWait(self.driver, 10).until(
-                        lambda driver: driver.execute_script("return document.readyState") == "complete"
+                        lambda driver: driver.execute_script(
+                            "return document.readyState"
+                        )
+                        == "complete"
                     )
-            
+
             # Optionally, wait for some condition after executing the JS code : Contributed by (https://github.com/jonymusky)
-            wait_for = kwargs.get('wait_for', False)
+            wait_for = kwargs.get("wait_for", False)
             if wait_for:
                 if callable(wait_for):
                     print("[LOG] 🔄 Waiting for condition...")
@@ -268,32 +293,37 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
                     print("[LOG] 🔄 Waiting for condition...")
                     WebDriverWait(self.driver, 20).until(
                         EC.presence_of_element_located((By.CSS_SELECTOR, wait_for))
-                    ) 
-            
+                    )
+
             if not can_not_be_done_headless:
                 html = sanitize_input_encode(self.driver.page_source)
-            self.driver = self.execute_hook('before_return_html', self.driver, html)
-            
+            self.driver = self.execute_hook("before_return_html", self.driver, html)
+
             # Store in cache
-            cache_file_path = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai", "cache", url_hash)
+            cache_file_path = os.path.join(
+                os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()),
+                ".crawl4ai",
+                "cache",
+                url_hash,
+            )
             with open(cache_file_path, "w", encoding="utf-8") as f:
                 f.write(html)
-                
+
             if self.verbose:
                 print(f"[LOG] ✅ Crawled {url} successfully!")
-            
+
             return html
         except InvalidArgumentException as e:
-            if not hasattr(e, 'msg'):
+            if not hasattr(e, "msg"):
                 e.msg = sanitize_input_encode(str(e))
             raise InvalidArgumentException(f"Failed to crawl {url}: {e.msg}")
         except WebDriverException as e:
             # If e does nlt have msg attribute create it and set it to str(e)
-            if not hasattr(e, 'msg'):
+            if not hasattr(e, "msg"):
                 e.msg = sanitize_input_encode(str(e))
-            raise WebDriverException(f"Failed to crawl {url}: {e.msg}")  
+            raise WebDriverException(f"Failed to crawl {url}: {e.msg}")
         except Exception as e:
-            if not hasattr(e, 'msg'):
+            if not hasattr(e, "msg"):
                 e.msg = sanitize_input_encode(str(e))
             raise Exception(f"Failed to crawl {url}: {e.msg}")
 
@@ -301,7 +331,9 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         try:
             # Get the dimensions of the page
             total_width = self.driver.execute_script("return document.body.scrollWidth")
-            total_height = self.driver.execute_script("return document.body.scrollHeight")
+            total_height = self.driver.execute_script(
+                "return document.body.scrollHeight"
+            )
 
             # Set the window size to the dimensions of the page
             self.driver.set_window_size(total_width, total_height)
@@ -313,25 +345,27 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
             image = Image.open(BytesIO(screenshot))
 
             # Convert image to RGB mode (this will handle both RGB and RGBA images)
-            rgb_image = image.convert('RGB')
+            rgb_image = image.convert("RGB")
 
             # Convert to JPEG and compress
             buffered = BytesIO()
             rgb_image.save(buffered, format="JPEG", quality=85)
-            img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
+            img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
 
             if self.verbose:
-                print(f"[LOG] 📸 Screenshot taken and converted to base64")
+                print("[LOG] 📸 Screenshot taken and converted to base64")
 
             return img_base64
         except Exception as e:
-            error_message = sanitize_input_encode(f"Failed to take screenshot: {str(e)}")
+            error_message = sanitize_input_encode(
+                f"Failed to take screenshot: {str(e)}"
+            )
             print(error_message)
 
             # Generate an image with black background
-            img = Image.new('RGB', (800, 600), color='black')
+            img = Image.new("RGB", (800, 600), color="black")
             draw = ImageDraw.Draw(img)
-            
+
             # Load a font
             try:
                 font = ImageFont.truetype("arial.ttf", 40)
@@ -345,16 +379,16 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
 
             # Calculate text position
             text_position = (10, 10)
-            
+
             # Draw the text on the image
             draw.text(text_position, wrapped_text, fill=text_color, font=font)
-            
+
             # Convert to base64
             buffered = BytesIO()
             img.save(buffered, format="JPEG")
-            img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
+            img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
 
             return img_base64
-        
+
     def quit(self):
         self.driver.quit()

crawl4ai/legacy/database.py

@@ -7,11 +7,13 @@ DB_PATH = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".cra
 os.makedirs(DB_PATH, exist_ok=True)
 DB_PATH = os.path.join(DB_PATH, "crawl4ai.db")
 
+
 def init_db():
     global DB_PATH
     conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
-    cursor.execute('''
+    cursor.execute(
+        """
         CREATE TABLE IF NOT EXISTS crawled_data (
             url TEXT PRIMARY KEY,
             html TEXT,
@@ -24,31 +26,42 @@ def init_db():
             metadata TEXT DEFAULT "{}",
             screenshot TEXT DEFAULT ""
         )
-    ''')
+    """
+    )
     conn.commit()
     conn.close()
 
+
 def alter_db_add_screenshot(new_column: str = "media"):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""')
+        cursor.execute(
+            f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""'
+        )
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error altering database to add screenshot column: {e}")
 
+
 def check_db_path():
     if not DB_PATH:
         raise ValueError("Database path is not set or is empty.")
 
-def get_cached_url(url: str) -> Optional[Tuple[str, str, str, str, str, str, str, bool, str]]:
+
+def get_cached_url(
+    url: str,
+) -> Optional[Tuple[str, str, str, str, str, str, str, bool, str]]:
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('SELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?', (url,))
+        cursor.execute(
+            "SELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?",
+            (url,),
+        )
         result = cursor.fetchone()
         conn.close()
         return result
@@ -56,12 +69,25 @@ def get_cached_url(url: str) -> Optional[Tuple[str, str, str, str, str, str, str
         print(f"Error retrieving cached URL: {e}")
         return None
 
-def cache_url(url: str, html: str, cleaned_html: str, markdown: str, extracted_content: str, success: bool, media : str = "{}", links : str = "{}", metadata : str = "{}", screenshot: str = ""):
+
+def cache_url(
+    url: str,
+    html: str,
+    cleaned_html: str,
+    markdown: str,
+    extracted_content: str,
+    success: bool,
+    media: str = "{}",
+    links: str = "{}",
+    metadata: str = "{}",
+    screenshot: str = "",
+):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('''
+        cursor.execute(
+            """
             INSERT INTO crawled_data (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ON CONFLICT(url) DO UPDATE SET
@@ -74,18 +100,32 @@ def cache_url(url: str, html: str, cleaned_html: str, markdown: str, extracted_c
                 links = excluded.links,    
                 metadata = excluded.metadata,      
                 screenshot = excluded.screenshot
-        ''', (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot))
+        """,
+            (
+                url,
+                html,
+                cleaned_html,
+                markdown,
+                extracted_content,
+                success,
+                media,
+                links,
+                metadata,
+                screenshot,
+            ),
+        )
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error caching URL: {e}")
 
+
 def get_total_count() -> int:
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('SELECT COUNT(*) FROM crawled_data')
+        cursor.execute("SELECT COUNT(*) FROM crawled_data")
         result = cursor.fetchone()
         conn.close()
         return result[0]
@@ -93,43 +133,48 @@ def get_total_count() -> int:
         print(f"Error getting total count: {e}")
         return 0
 
+
 def clear_db():
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('DELETE FROM crawled_data')
+        cursor.execute("DELETE FROM crawled_data")
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error clearing database: {e}")
-        
+
+
 def flush_db():
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute('DROP TABLE crawled_data')
+        cursor.execute("DROP TABLE crawled_data")
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error flushing database: {e}")
 
+
 def update_existing_records(new_column: str = "media", default_value: str = "{}"):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute(f'UPDATE crawled_data SET {new_column} = "{default_value}" WHERE screenshot IS NULL')
+        cursor.execute(
+            f'UPDATE crawled_data SET {new_column} = "{default_value}" WHERE screenshot IS NULL'
+        )
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error updating existing records: {e}")
 
+
 if __name__ == "__main__":
     # Delete the existing database file
     if os.path.exists(DB_PATH):
         os.remove(DB_PATH)
-    init_db()  
+    init_db()
     # alter_db_add_screenshot("COL_NAME")
-    

crawl4ai/legacy/docs_manager.py

@@ -0,0 +1,75 @@
+import requests
+import shutil
+from pathlib import Path
+from crawl4ai.async_logger import AsyncLogger
+from crawl4ai.llmtxt import AsyncLLMTextManager
+
+
+class DocsManager:
+    def __init__(self, logger=None):
+        self.docs_dir = Path.home() / ".crawl4ai" / "docs"
+        self.local_docs = Path(__file__).parent.parent / "docs" / "llm.txt"
+        self.docs_dir.mkdir(parents=True, exist_ok=True)
+        self.logger = logger or AsyncLogger(verbose=True)
+        self.llm_text = AsyncLLMTextManager(self.docs_dir, self.logger)
+
+    async def ensure_docs_exist(self):
+        """Fetch docs if not present"""
+        if not any(self.docs_dir.iterdir()):
+            await self.fetch_docs()
+
+    async def fetch_docs(self) -> bool:
+        """Copy from local docs or download from GitHub"""
+        try:
+            # Try local first
+            if self.local_docs.exists() and (
+                any(self.local_docs.glob("*.md"))
+                or any(self.local_docs.glob("*.tokens"))
+            ):
+                # Empty the local docs directory
+                for file_path in self.docs_dir.glob("*.md"):
+                    file_path.unlink()
+                # for file_path in self.docs_dir.glob("*.tokens"):
+                #     file_path.unlink()
+                for file_path in self.local_docs.glob("*.md"):
+                    shutil.copy2(file_path, self.docs_dir / file_path.name)
+                # for file_path in self.local_docs.glob("*.tokens"):
+                #     shutil.copy2(file_path, self.docs_dir / file_path.name)
+                return True
+
+            # Fallback to GitHub
+            response = requests.get(
+                "https://api.github.com/repos/unclecode/crawl4ai/contents/docs/llm.txt",
+                headers={"Accept": "application/vnd.github.v3+json"},
+            )
+            response.raise_for_status()
+
+            for item in response.json():
+                if item["type"] == "file" and item["name"].endswith(".md"):
+                    content = requests.get(item["download_url"]).text
+                    with open(self.docs_dir / item["name"], "w", encoding="utf-8") as f:
+                        f.write(content)
+            return True
+
+        except Exception as e:
+            self.logger.error(f"Failed to fetch docs: {str(e)}")
+            raise
+
+    def list(self) -> list[str]:
+        """List available topics"""
+        names = [file_path.stem for file_path in self.docs_dir.glob("*.md")]
+        # Remove [0-9]+_ prefix
+        names = [name.split("_", 1)[1] if name[0].isdigit() else name for name in names]
+        # Exclude those end with .xs.md and .q.md
+        names = [
+            name
+            for name in names
+            if not name.endswith(".xs") and not name.endswith(".q")
+        ]
+        return names
+
+    def generate(self, sections, mode="extended"):
+        return self.llm_text.generate(sections, mode)
+
+    def search(self, query: str, top_k: int = 5):
+        return self.llm_text.search(query, top_k)

crawl4ai/legacy/llmtxt.py

@@ -0,0 +1,546 @@
+import os
+from pathlib import Path
+import re
+from typing import Dict, List, Tuple, Optional, Any
+import json
+from tqdm import tqdm
+import time
+import psutil
+import numpy as np
+from rank_bm25 import BM25Okapi
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from litellm import batch_completion
+from .async_logger import AsyncLogger
+import litellm
+import pickle
+import hashlib  # <--- ADDED for file-hash
+import glob
+
+litellm.set_verbose = False
+
+
+def _compute_file_hash(file_path: Path) -> str:
+    """Compute MD5 hash for the file's entire content."""
+    hash_md5 = hashlib.md5()
+    with file_path.open("rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
+class AsyncLLMTextManager:
+    def __init__(
+        self,
+        docs_dir: Path,
+        logger: Optional[AsyncLogger] = None,
+        max_concurrent_calls: int = 5,
+        batch_size: int = 3,
+    ) -> None:
+        self.docs_dir = docs_dir
+        self.logger = logger
+        self.max_concurrent_calls = max_concurrent_calls
+        self.batch_size = batch_size
+        self.bm25_index = None
+        self.document_map: Dict[str, Any] = {}
+        self.tokenized_facts: List[str] = []
+        self.bm25_index_file = self.docs_dir / "bm25_index.pkl"
+
+    async def _process_document_batch(self, doc_batch: List[Path]) -> None:
+        """Process a batch of documents in parallel"""
+        contents = []
+        for file_path in doc_batch:
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    contents.append(f.read())
+            except Exception as e:
+                self.logger.error(f"Error reading {file_path}: {str(e)}")
+                contents.append("")  # Add empty content to maintain batch alignment
+
+        prompt = """Given a documentation file, generate a list of atomic facts where each fact:
+1. Represents a single piece of knowledge
+2. Contains variations in terminology for the same concept
+3. References relevant code patterns if they exist
+4. Is written in a way that would match natural language queries
+
+Each fact should follow this format:
+<main_concept>: <fact_statement> | <related_terms> | <code_reference>
+
+Example Facts:
+browser_config: Configure headless mode and browser type for AsyncWebCrawler | headless, browser_type, chromium, firefox | BrowserConfig(browser_type="chromium", headless=True)
+redis_connection: Redis client connection requires host and port configuration | redis setup, redis client, connection params | Redis(host='localhost', port=6379, db=0)
+pandas_filtering: Filter DataFrame rows using boolean conditions | dataframe filter, query, boolean indexing | df[df['column'] > 5]
+
+Wrap your response in <index>...</index> tags.
+"""
+
+        # Prepare messages for batch processing
+        messages_list = [
+            [
+                {
+                    "role": "user",
+                    "content": f"{prompt}\n\nGenerate index for this documentation:\n\n{content}",
+                }
+            ]
+            for content in contents
+            if content
+        ]
+
+        try:
+            responses = batch_completion(
+                model="anthropic/claude-3-5-sonnet-latest",
+                messages=messages_list,
+                logger_fn=None,
+            )
+
+            # Process responses and save index files
+            for response, file_path in zip(responses, doc_batch):
+                try:
+                    index_content_match = re.search(
+                        r"<index>(.*?)</index>",
+                        response.choices[0].message.content,
+                        re.DOTALL,
+                    )
+                    if not index_content_match:
+                        self.logger.warning(
+                            f"No <index>...</index> content found for {file_path}"
+                        )
+                        continue
+
+                    index_content = re.sub(
+                        r"\n\s*\n", "\n", index_content_match.group(1)
+                    ).strip()
+                    if index_content:
+                        index_file = file_path.with_suffix(".q.md")
+                        with open(index_file, "w", encoding="utf-8") as f:
+                            f.write(index_content)
+                        self.logger.info(f"Created index file: {index_file}")
+                    else:
+                        self.logger.warning(
+                            f"No index content found in response for {file_path}"
+                        )
+
+                except Exception as e:
+                    self.logger.error(
+                        f"Error processing response for {file_path}: {str(e)}"
+                    )
+
+        except Exception as e:
+            self.logger.error(f"Error in batch completion: {str(e)}")
+
+    def _validate_fact_line(self, line: str) -> Tuple[bool, Optional[str]]:
+        if "|" not in line:
+            return False, "Missing separator '|'"
+
+        parts = [p.strip() for p in line.split("|")]
+        if len(parts) != 3:
+            return False, f"Expected 3 parts, got {len(parts)}"
+
+        concept_part = parts[0]
+        if ":" not in concept_part:
+            return False, "Missing ':' in concept definition"
+
+        return True, None
+
+    def _load_or_create_token_cache(self, fact_file: Path) -> Dict:
+        """
+        Load token cache from .q.tokens if present and matching file hash.
+        Otherwise return a new structure with updated file-hash.
+        """
+        cache_file = fact_file.with_suffix(".q.tokens")
+        current_hash = _compute_file_hash(fact_file)
+
+        if cache_file.exists():
+            try:
+                with open(cache_file, "r") as f:
+                    cache = json.load(f)
+                # If the hash matches, return it directly
+                if cache.get("content_hash") == current_hash:
+                    return cache
+                # Otherwise, we signal that it's changed
+                self.logger.info(f"Hash changed for {fact_file}, reindex needed.")
+            except json.JSONDecodeError:
+                self.logger.warning(f"Corrupt token cache for {fact_file}, rebuilding.")
+            except Exception as e:
+                self.logger.warning(f"Error reading cache for {fact_file}: {str(e)}")
+
+        # Return a fresh cache
+        return {"facts": {}, "content_hash": current_hash}
+
+    def _save_token_cache(self, fact_file: Path, cache: Dict) -> None:
+        cache_file = fact_file.with_suffix(".q.tokens")
+        # Always ensure we're saving the correct file-hash
+        cache["content_hash"] = _compute_file_hash(fact_file)
+        with open(cache_file, "w") as f:
+            json.dump(cache, f)
+
+    def preprocess_text(self, text: str) -> List[str]:
+        parts = [x.strip() for x in text.split("|")] if "|" in text else [text]
+        # Remove : after the first word of parts[0]
+        parts[0] = re.sub(r"^(.*?):", r"\1", parts[0])
+
+        lemmatizer = WordNetLemmatizer()
+        stop_words = set(stopwords.words("english")) - {
+            "how",
+            "what",
+            "when",
+            "where",
+            "why",
+            "which",
+        }
+
+        tokens = []
+        for part in parts:
+            if "(" in part and ")" in part:
+                code_tokens = re.findall(
+                    r'[\w_]+(?=\()|[\w_]+(?==[\'"]{1}[\w_]+[\'"]{1})', part
+                )
+                tokens.extend(code_tokens)
+
+            words = word_tokenize(part.lower())
+            tokens.extend(
+                [
+                    lemmatizer.lemmatize(token)
+                    for token in words
+                    if token not in stop_words
+                ]
+            )
+
+        return tokens
+
+    def maybe_load_bm25_index(self, clear_cache=False) -> bool:
+        """
+        Load existing BM25 index from disk, if present and clear_cache=False.
+        """
+        if not clear_cache and os.path.exists(self.bm25_index_file):
+            self.logger.info("Loading existing BM25 index from disk.")
+            with open(self.bm25_index_file, "rb") as f:
+                data = pickle.load(f)
+            self.tokenized_facts = data["tokenized_facts"]
+            self.bm25_index = data["bm25_index"]
+            return True
+        return False
+
+    def build_search_index(self, clear_cache=False) -> None:
+        """
+        Checks for new or modified .q.md files by comparing file-hash.
+        If none need reindexing and clear_cache is False, loads existing index if available.
+        Otherwise, reindexes only changed/new files and merges or creates a new index.
+        """
+        # If clear_cache is True, we skip partial logic: rebuild everything from scratch
+        if clear_cache:
+            self.logger.info("Clearing cache and rebuilding full search index.")
+            if self.bm25_index_file.exists():
+                self.bm25_index_file.unlink()
+
+        process = psutil.Process()
+        self.logger.info("Checking which .q.md files need (re)indexing...")
+
+        # Gather all .q.md files
+        q_files = [
+            self.docs_dir / f for f in os.listdir(self.docs_dir) if f.endswith(".q.md")
+        ]
+
+        # We'll store known (unchanged) facts in these lists
+        existing_facts: List[str] = []
+        existing_tokens: List[List[str]] = []
+
+        # Keep track of invalid lines for logging
+        invalid_lines = []
+        needSet = []  # files that must be (re)indexed
+
+        for qf in q_files:
+            token_cache_file = qf.with_suffix(".q.tokens")
+
+            # If no .q.tokens or clear_cache is True → definitely reindex
+            if clear_cache or not token_cache_file.exists():
+                needSet.append(qf)
+                continue
+
+            # Otherwise, load the existing cache and compare hash
+            cache = self._load_or_create_token_cache(qf)
+            # If the .q.tokens was out of date (i.e. changed hash), we reindex
+            if len(cache["facts"]) == 0 or cache.get(
+                "content_hash"
+            ) != _compute_file_hash(qf):
+                needSet.append(qf)
+            else:
+                # File is unchanged → retrieve cached token data
+                for line, cache_data in cache["facts"].items():
+                    existing_facts.append(line)
+                    existing_tokens.append(cache_data["tokens"])
+                    self.document_map[line] = qf  # track the doc for that fact
+
+        if not needSet and not clear_cache:
+            # If no file needs reindexing, try loading existing index
+            if self.maybe_load_bm25_index(clear_cache=False):
+                self.logger.info(
+                    "No new/changed .q.md files found. Using existing BM25 index."
+                )
+                return
+            else:
+                # If there's no existing index, we must build a fresh index from the old caches
+                self.logger.info(
+                    "No existing BM25 index found. Building from cached facts."
+                )
+                if existing_facts:
+                    self.logger.info(
+                        f"Building BM25 index with {len(existing_facts)} cached facts."
+                    )
+                    self.bm25_index = BM25Okapi(existing_tokens)
+                    self.tokenized_facts = existing_facts
+                    with open(self.bm25_index_file, "wb") as f:
+                        pickle.dump(
+                            {
+                                "bm25_index": self.bm25_index,
+                                "tokenized_facts": self.tokenized_facts,
+                            },
+                            f,
+                        )
+                else:
+                    self.logger.warning("No facts found at all. Index remains empty.")
+                return
+
+        # ----------------------------------------------------- /Users/unclecode/.crawl4ai/docs/14_proxy_security.q.q.tokens '/Users/unclecode/.crawl4ai/docs/14_proxy_security.q.md'
+        # If we reach here, we have new or changed .q.md files
+        # We'll parse them, reindex them, and then combine with existing_facts
+        # -----------------------------------------------------
+
+        self.logger.info(f"{len(needSet)} file(s) need reindexing. Parsing now...")
+
+        # 1) Parse the new or changed .q.md files
+        new_facts = []
+        new_tokens = []
+        with tqdm(total=len(needSet), desc="Indexing changed files") as file_pbar:
+            for file in needSet:
+                # We'll build up a fresh cache
+                fresh_cache = {"facts": {}, "content_hash": _compute_file_hash(file)}
+                try:
+                    with open(file, "r", encoding="utf-8") as f_obj:
+                        content = f_obj.read().strip()
+                        lines = [l.strip() for l in content.split("\n") if l.strip()]
+
+                    for line in lines:
+                        is_valid, error = self._validate_fact_line(line)
+                        if not is_valid:
+                            invalid_lines.append((file, line, error))
+                            continue
+
+                        tokens = self.preprocess_text(line)
+                        fresh_cache["facts"][line] = {
+                            "tokens": tokens,
+                            "added": time.time(),
+                        }
+                        new_facts.append(line)
+                        new_tokens.append(tokens)
+                        self.document_map[line] = file
+
+                    # Save the new .q.tokens with updated hash
+                    self._save_token_cache(file, fresh_cache)
+
+                    mem_usage = process.memory_info().rss / 1024 / 1024
+                    self.logger.debug(
+                        f"Memory usage after {file.name}: {mem_usage:.2f}MB"
+                    )
+
+                except Exception as e:
+                    self.logger.error(f"Error processing {file}: {str(e)}")
+
+                file_pbar.update(1)
+
+        if invalid_lines:
+            self.logger.warning(f"Found {len(invalid_lines)} invalid fact lines:")
+            for file, line, error in invalid_lines:
+                self.logger.warning(f"{file}: {error} in line: {line[:50]}...")
+
+        # 2) Merge newly tokenized facts with the existing ones
+        all_facts = existing_facts + new_facts
+        all_tokens = existing_tokens + new_tokens
+
+        # 3) Build BM25 index from combined facts
+        self.logger.info(
+            f"Building BM25 index with {len(all_facts)} total facts (old + new)."
+        )
+        self.bm25_index = BM25Okapi(all_tokens)
+        self.tokenized_facts = all_facts
+
+        # 4) Save the updated BM25 index to disk
+        with open(self.bm25_index_file, "wb") as f:
+            pickle.dump(
+                {
+                    "bm25_index": self.bm25_index,
+                    "tokenized_facts": self.tokenized_facts,
+                },
+                f,
+            )
+
+        final_mem = process.memory_info().rss / 1024 / 1024
+        self.logger.info(f"Search index updated. Final memory usage: {final_mem:.2f}MB")
+
+    async def generate_index_files(
+        self, force_generate_facts: bool = False, clear_bm25_cache: bool = False
+    ) -> None:
+        """
+        Generate index files for all documents in parallel batches
+
+        Args:
+            force_generate_facts (bool): If True, regenerate indexes even if they exist
+            clear_bm25_cache (bool): If True, clear existing BM25 index cache
+        """
+        self.logger.info("Starting index generation for documentation files.")
+
+        md_files = [
+            self.docs_dir / f
+            for f in os.listdir(self.docs_dir)
+            if f.endswith(".md") and not any(f.endswith(x) for x in [".q.md", ".xs.md"])
+        ]
+
+        # Filter out files that already have .q files unless force=True
+        if not force_generate_facts:
+            md_files = [
+                f
+                for f in md_files
+                if not (self.docs_dir / f.name.replace(".md", ".q.md")).exists()
+            ]
+
+        if not md_files:
+            self.logger.info("All index files exist. Use force=True to regenerate.")
+        else:
+            # Process documents in batches
+            for i in range(0, len(md_files), self.batch_size):
+                batch = md_files[i : i + self.batch_size]
+                self.logger.info(
+                    f"Processing batch {i//self.batch_size + 1}/{(len(md_files)//self.batch_size) + 1}"
+                )
+                await self._process_document_batch(batch)
+
+        self.logger.info("Index generation complete, building/updating search index.")
+        self.build_search_index(clear_cache=clear_bm25_cache)
+
+    def generate(self, sections: List[str], mode: str = "extended") -> str:
+        # Get all markdown files
+        all_files = glob.glob(str(self.docs_dir / "[0-9]*.md")) + glob.glob(
+            str(self.docs_dir / "[0-9]*.xs.md")
+        )
+
+        # Extract base names without extensions
+        base_docs = {
+            Path(f).name.split(".")[0]
+            for f in all_files
+            if not Path(f).name.endswith(".q.md")
+        }
+
+        # Filter by sections if provided
+        if sections:
+            base_docs = {
+                doc
+                for doc in base_docs
+                if any(section.lower() in doc.lower() for section in sections)
+            }
+
+        # Get file paths based on mode
+        files = []
+        for doc in sorted(
+            base_docs,
+            key=lambda x: int(x.split("_")[0]) if x.split("_")[0].isdigit() else 999999,
+        ):
+            if mode == "condensed":
+                xs_file = self.docs_dir / f"{doc}.xs.md"
+                regular_file = self.docs_dir / f"{doc}.md"
+                files.append(str(xs_file if xs_file.exists() else regular_file))
+            else:
+                files.append(str(self.docs_dir / f"{doc}.md"))
+
+        # Read and format content
+        content = []
+        for file in files:
+            try:
+                with open(file, "r", encoding="utf-8") as f:
+                    fname = Path(file).name
+                    content.append(f"{'#'*20}\n# {fname}\n{'#'*20}\n\n{f.read()}")
+            except Exception as e:
+                self.logger.error(f"Error reading {file}: {str(e)}")
+
+        return "\n\n---\n\n".join(content) if content else ""
+
+    def search(self, query: str, top_k: int = 5) -> str:
+        if not self.bm25_index:
+            return "No search index available. Call build_search_index() first."
+
+        query_tokens = self.preprocess_text(query)
+        doc_scores = self.bm25_index.get_scores(query_tokens)
+
+        mean_score = np.mean(doc_scores)
+        std_score = np.std(doc_scores)
+        score_threshold = mean_score + (0.25 * std_score)
+
+        file_data = self._aggregate_search_scores(
+            doc_scores=doc_scores,
+            score_threshold=score_threshold,
+            query_tokens=query_tokens,
+        )
+
+        ranked_files = sorted(
+            file_data.items(),
+            key=lambda x: (
+                x[1]["code_match_score"] * 2.0
+                + x[1]["match_count"] * 1.5
+                + x[1]["total_score"]
+            ),
+            reverse=True,
+        )[:top_k]
+
+        results = []
+        for file, _ in ranked_files:
+            main_doc = str(file).replace(".q.md", ".md")
+            if os.path.exists(self.docs_dir / main_doc):
+                with open(self.docs_dir / main_doc, "r", encoding="utf-8") as f:
+                    only_file_name = main_doc.split("/")[-1]
+                    content = ["#" * 20, f"# {only_file_name}", "#" * 20, "", f.read()]
+                    results.append("\n".join(content))
+
+        return "\n\n---\n\n".join(results)
+
+    def _aggregate_search_scores(
+        self, doc_scores: List[float], score_threshold: float, query_tokens: List[str]
+    ) -> Dict:
+        file_data = {}
+
+        for idx, score in enumerate(doc_scores):
+            if score <= score_threshold:
+                continue
+
+            fact = self.tokenized_facts[idx]
+            file_path = self.document_map[fact]
+
+            if file_path not in file_data:
+                file_data[file_path] = {
+                    "total_score": 0,
+                    "match_count": 0,
+                    "code_match_score": 0,
+                    "matched_facts": [],
+                }
+
+            components = fact.split("|") if "|" in fact else [fact]
+
+            code_match_score = 0
+            if len(components) == 3:
+                code_ref = components[2].strip()
+                code_tokens = self.preprocess_text(code_ref)
+                code_match_score = len(set(query_tokens) & set(code_tokens)) / len(
+                    query_tokens
+                )
+
+            file_data[file_path]["total_score"] += score
+            file_data[file_path]["match_count"] += 1
+            file_data[file_path]["code_match_score"] = max(
+                file_data[file_path]["code_match_score"], code_match_score
+            )
+            file_data[file_path]["matched_facts"].append(fact)
+
+        return file_data
+
+    def refresh_index(self) -> None:
+        """Convenience method for a full rebuild."""
+        self.build_search_index(clear_cache=True)

crawl4ai/legacy/version_manager.py

@@ -1,14 +1,14 @@
 # version_manager.py
-import os
 from pathlib import Path
 from packaging import version
 from . import __version__
 
+
 class VersionManager:
     def __init__(self):
         self.home_dir = Path.home() / ".crawl4ai"
         self.version_file = self.home_dir / "version.txt"
-        
+
     def get_installed_version(self):
         """Get the version recorded in home directory"""
         if not self.version_file.exists():
@@ -17,14 +17,13 @@ class VersionManager:
             return version.parse(self.version_file.read_text().strip())
         except:
             return None
-            
+
     def update_version(self):
         """Update the version file to current library version"""
         self.version_file.write_text(__version__.__version__)
-        
+
     def needs_update(self):
         """Check if database needs update based on version"""
         installed = self.get_installed_version()
         current = version.parse(__version__.__version__)
         return installed is None or installed < current
-

crawl4ai/legacy/web_crawler.py

@@ -0,0 +1,294 @@
+import os, time
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+from pathlib import Path
+
+from .models import UrlModel, CrawlResult
+from .database import init_db, get_cached_url, cache_url
+from .utils import *
+from .chunking_strategy import *
+from .extraction_strategy import *
+from .crawler_strategy import *
+from typing import List
+from concurrent.futures import ThreadPoolExecutor
+from .content_scraping_strategy import WebScrapingStrategy
+from .config import *
+import warnings
+import json
+
+warnings.filterwarnings(
+    "ignore",
+    message='Field "model_name" has conflict with protected namespace "model_".',
+)
+
+
+class WebCrawler:
+    def __init__(
+        self,
+        crawler_strategy: CrawlerStrategy = None,
+        always_by_pass_cache: bool = False,
+        verbose: bool = False,
+    ):
+        self.crawler_strategy = crawler_strategy or LocalSeleniumCrawlerStrategy(
+            verbose=verbose
+        )
+        self.always_by_pass_cache = always_by_pass_cache
+        self.crawl4ai_folder = os.path.join(
+            os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
+        )
+        os.makedirs(self.crawl4ai_folder, exist_ok=True)
+        os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True)
+        init_db()
+        self.ready = False
+
+    def warmup(self):
+        print("[LOG] 🌤️  Warming up the WebCrawler")
+        self.run(
+            url="https://google.com/",
+            word_count_threshold=5,
+            extraction_strategy=NoExtractionStrategy(),
+            bypass_cache=False,
+            verbose=False,
+        )
+        self.ready = True
+        print("[LOG] 🌞 WebCrawler is ready to crawl")
+
+    def fetch_page(
+        self,
+        url_model: UrlModel,
+        provider: str = DEFAULT_PROVIDER,
+        api_token: str = None,
+        extract_blocks_flag: bool = True,
+        word_count_threshold=MIN_WORD_THRESHOLD,
+        css_selector: str = None,
+        screenshot: bool = False,
+        use_cached_html: bool = False,
+        extraction_strategy: ExtractionStrategy = None,
+        chunking_strategy: ChunkingStrategy = RegexChunking(),
+        **kwargs,
+    ) -> CrawlResult:
+        return self.run(
+            url_model.url,
+            word_count_threshold,
+            extraction_strategy or NoExtractionStrategy(),
+            chunking_strategy,
+            bypass_cache=url_model.forced,
+            css_selector=css_selector,
+            screenshot=screenshot,
+            **kwargs,
+        )
+        pass
+
+    def fetch_pages(
+        self,
+        url_models: List[UrlModel],
+        provider: str = DEFAULT_PROVIDER,
+        api_token: str = None,
+        extract_blocks_flag: bool = True,
+        word_count_threshold=MIN_WORD_THRESHOLD,
+        use_cached_html: bool = False,
+        css_selector: str = None,
+        screenshot: bool = False,
+        extraction_strategy: ExtractionStrategy = None,
+        chunking_strategy: ChunkingStrategy = RegexChunking(),
+        **kwargs,
+    ) -> List[CrawlResult]:
+        extraction_strategy = extraction_strategy or NoExtractionStrategy()
+
+        def fetch_page_wrapper(url_model, *args, **kwargs):
+            return self.fetch_page(url_model, *args, **kwargs)
+
+        with ThreadPoolExecutor() as executor:
+            results = list(
+                executor.map(
+                    fetch_page_wrapper,
+                    url_models,
+                    [provider] * len(url_models),
+                    [api_token] * len(url_models),
+                    [extract_blocks_flag] * len(url_models),
+                    [word_count_threshold] * len(url_models),
+                    [css_selector] * len(url_models),
+                    [screenshot] * len(url_models),
+                    [use_cached_html] * len(url_models),
+                    [extraction_strategy] * len(url_models),
+                    [chunking_strategy] * len(url_models),
+                    *[kwargs] * len(url_models),
+                )
+            )
+
+        return results
+
+    def run(
+        self,
+        url: str,
+        word_count_threshold=MIN_WORD_THRESHOLD,
+        extraction_strategy: ExtractionStrategy = None,
+        chunking_strategy: ChunkingStrategy = RegexChunking(),
+        bypass_cache: bool = False,
+        css_selector: str = None,
+        screenshot: bool = False,
+        user_agent: str = None,
+        verbose=True,
+        **kwargs,
+    ) -> CrawlResult:
+        try:
+            extraction_strategy = extraction_strategy or NoExtractionStrategy()
+            extraction_strategy.verbose = verbose
+            if not isinstance(extraction_strategy, ExtractionStrategy):
+                raise ValueError("Unsupported extraction strategy")
+            if not isinstance(chunking_strategy, ChunkingStrategy):
+                raise ValueError("Unsupported chunking strategy")
+
+            word_count_threshold = max(word_count_threshold, MIN_WORD_THRESHOLD)
+
+            cached = None
+            screenshot_data = None
+            extracted_content = None
+            if not bypass_cache and not self.always_by_pass_cache:
+                cached = get_cached_url(url)
+
+            if kwargs.get("warmup", True) and not self.ready:
+                return None
+
+            if cached:
+                html = sanitize_input_encode(cached[1])
+                extracted_content = sanitize_input_encode(cached[4])
+                if screenshot:
+                    screenshot_data = cached[9]
+                    if not screenshot_data:
+                        cached = None
+
+            if not cached or not html:
+                if user_agent:
+                    self.crawler_strategy.update_user_agent(user_agent)
+                t1 = time.time()
+                html = sanitize_input_encode(self.crawler_strategy.crawl(url, **kwargs))
+                t2 = time.time()
+                if verbose:
+                    print(
+                        f"[LOG] 🚀 Crawling done for {url}, success: {bool(html)}, time taken: {t2 - t1:.2f} seconds"
+                    )
+                if screenshot:
+                    screenshot_data = self.crawler_strategy.take_screenshot()
+
+            crawl_result = self.process_html(
+                url,
+                html,
+                extracted_content,
+                word_count_threshold,
+                extraction_strategy,
+                chunking_strategy,
+                css_selector,
+                screenshot_data,
+                verbose,
+                bool(cached),
+                **kwargs,
+            )
+            crawl_result.success = bool(html)
+            return crawl_result
+        except Exception as e:
+            if not hasattr(e, "msg"):
+                e.msg = str(e)
+            print(f"[ERROR] 🚫 Failed to crawl {url}, error: {e.msg}")
+            return CrawlResult(url=url, html="", success=False, error_message=e.msg)
+
+    def process_html(
+        self,
+        url: str,
+        html: str,
+        extracted_content: str,
+        word_count_threshold: int,
+        extraction_strategy: ExtractionStrategy,
+        chunking_strategy: ChunkingStrategy,
+        css_selector: str,
+        screenshot: bool,
+        verbose: bool,
+        is_cached: bool,
+        **kwargs,
+    ) -> CrawlResult:
+        t = time.time()
+        # Extract content from HTML
+        try:
+            t1 = time.time()
+            scrapping_strategy = WebScrapingStrategy()
+            extra_params = {
+                k: v
+                for k, v in kwargs.items()
+                if k not in ["only_text", "image_description_min_word_threshold"]
+            }
+            result = scrapping_strategy.scrap(
+                url,
+                html,
+                word_count_threshold=word_count_threshold,
+                css_selector=css_selector,
+                only_text=kwargs.get("only_text", False),
+                image_description_min_word_threshold=kwargs.get(
+                    "image_description_min_word_threshold",
+                    IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
+                ),
+                **extra_params,
+            )
+
+            # result = get_content_of_website_optimized(url, html, word_count_threshold, css_selector=css_selector, only_text=kwargs.get("only_text", False))
+            if verbose:
+                print(
+                    f"[LOG] 🚀 Content extracted for {url}, success: True, time taken: {time.time() - t1:.2f} seconds"
+                )
+
+            if result is None:
+                raise ValueError(f"Failed to extract content from the website: {url}")
+        except InvalidCSSSelectorError as e:
+            raise ValueError(str(e))
+
+        cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
+        markdown = sanitize_input_encode(result.get("markdown", ""))
+        media = result.get("media", [])
+        links = result.get("links", [])
+        metadata = result.get("metadata", {})
+
+        if extracted_content is None:
+            if verbose:
+                print(
+                    f"[LOG] 🔥 Extracting semantic blocks for {url}, Strategy: {extraction_strategy.name}"
+                )
+
+            sections = chunking_strategy.chunk(markdown)
+            extracted_content = extraction_strategy.run(url, sections)
+            extracted_content = json.dumps(
+                extracted_content, indent=4, default=str, ensure_ascii=False
+            )
+
+            if verbose:
+                print(
+                    f"[LOG] 🚀 Extraction done for {url}, time taken: {time.time() - t:.2f} seconds."
+                )
+
+        screenshot = None if not screenshot else screenshot
+
+        if not is_cached:
+            cache_url(
+                url,
+                html,
+                cleaned_html,
+                markdown,
+                extracted_content,
+                True,
+                json.dumps(media),
+                json.dumps(links),
+                json.dumps(metadata),
+                screenshot=screenshot,
+            )
+
+        return CrawlResult(
+            url=url,
+            html=html,
+            cleaned_html=format_html(cleaned_html),
+            markdown=markdown,
+            media=media,
+            links=links,
+            metadata=metadata,
+            screenshot=screenshot,
+            extracted_content=extracted_content,
+            success=True,
+            error_message="",
+        )

crawl4ai/markdown_generation_strategy.py

@@ -1,124 +1,256 @@
 from abc import ABC, abstractmethod
+from tabnanny import verbose
 from typing import Optional, Dict, Any, Tuple
 from .models import MarkdownGenerationResult
-from .utils import CustomHTML2Text
-from .content_filter_strategy import RelevantContentFilter, BM25ContentFilter
+from .html2text import CustomHTML2Text
+from .content_filter_strategy import RelevantContentFilter
 import re
 from urllib.parse import urljoin
 
 # Pre-compile the regex pattern
 LINK_PATTERN = re.compile(r'!?\[([^\]]+)\]\(([^)]+?)(?:\s+"([^"]*)")?\)')
 
+
+def fast_urljoin(base: str, url: str) -> str:
+    """Fast URL joining for common cases."""
+    if url.startswith(("http://", "https://", "mailto:", "//")):
+        return url
+    if url.startswith("/"):
+        # Handle absolute paths
+        if base.endswith("/"):
+            return base[:-1] + url
+        return base + url
+    return urljoin(base, url)
+
+
 class MarkdownGenerationStrategy(ABC):
     """Abstract base class for markdown generation strategies."""
-    def __init__(self, content_filter: Optional[RelevantContentFilter] = None):
+
+    def __init__(
+        self,
+        content_filter: Optional[RelevantContentFilter] = None,
+        options: Optional[Dict[str, Any]] = None,
+        verbose: bool = False,
+    ):
         self.content_filter = content_filter
-    
+        self.options = options or {}
+        self.verbose = verbose
+
     @abstractmethod
-    def generate_markdown(self, 
-                         cleaned_html: str, 
-                         base_url: str = "",
-                         html2text_options: Optional[Dict[str, Any]] = None,
-                         content_filter: Optional[RelevantContentFilter] = None,
-                         citations: bool = True,
-                         **kwargs) -> MarkdownGenerationResult:
+    def generate_markdown(
+        self,
+        cleaned_html: str,
+        base_url: str = "",
+        html2text_options: Optional[Dict[str, Any]] = None,
+        content_filter: Optional[RelevantContentFilter] = None,
+        citations: bool = True,
+        **kwargs,
+    ) -> MarkdownGenerationResult:
         """Generate markdown from cleaned HTML."""
         pass
 
+
 class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
-    """Default implementation of markdown generation strategy."""
-    def __init__(self, content_filter: Optional[RelevantContentFilter] = None):
-        super().__init__(content_filter)
-    
-    def convert_links_to_citations(self, markdown: str, base_url: str = "") -> Tuple[str, str]:
+    """
+    Default implementation of markdown generation strategy.
+
+    How it works:
+    1. Generate raw markdown from cleaned HTML.
+    2. Convert links to citations.
+    3. Generate fit markdown if content filter is provided.
+    4. Return MarkdownGenerationResult.
+
+    Args:
+        content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
+        options (Optional[Dict[str, Any]]): Additional options for markdown generation. Defaults to None.
+
+    Returns:
+        MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
+    """
+
+    def __init__(
+        self,
+        content_filter: Optional[RelevantContentFilter] = None,
+        options: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__(content_filter, options)
+
+    def convert_links_to_citations(
+        self, markdown: str, base_url: str = ""
+    ) -> Tuple[str, str]:
+        """
+        Convert links in markdown to citations.
+
+        How it works:
+        1. Find all links in the markdown.
+        2. Convert links to citations.
+        3. Return converted markdown and references markdown.
+
+        Note:
+        This function uses a regex pattern to find links in markdown.
+
+        Args:
+            markdown (str): Markdown text.
+            base_url (str): Base URL for URL joins.
+
+        Returns:
+            Tuple[str, str]: Converted markdown and references markdown.
+        """
         link_map = {}
         url_cache = {}  # Cache for URL joins
         parts = []
         last_end = 0
         counter = 1
-        
+
         for match in LINK_PATTERN.finditer(markdown):
-            parts.append(markdown[last_end:match.start()])
+            parts.append(markdown[last_end : match.start()])
             text, url, title = match.groups()
-            
+
             # Use cached URL if available, otherwise compute and cache
-            if base_url and not url.startswith(('http://', 'https://', 'mailto:')):
+            if base_url and not url.startswith(("http://", "https://", "mailto:")):
                 if url not in url_cache:
                     url_cache[url] = fast_urljoin(base_url, url)
                 url = url_cache[url]
-                
+
             if url not in link_map:
                 desc = []
-                if title: desc.append(title)
-                if text and text != title: desc.append(text)
+                if title:
+                    desc.append(title)
+                if text and text != title:
+                    desc.append(text)
                 link_map[url] = (counter, ": " + " - ".join(desc) if desc else "")
                 counter += 1
-                
+
             num = link_map[url][0]
-            parts.append(f"{text}⟨{num}⟩" if not match.group(0).startswith('!') else f"![{text}⟨{num}⟩]")
+            parts.append(
+                f"{text}⟨{num}⟩"
+                if not match.group(0).startswith("!")
+                else f"![{text}⟨{num}⟩]"
+            )
             last_end = match.end()
-        
+
         parts.append(markdown[last_end:])
-        converted_text = ''.join(parts)
-        
+        converted_text = "".join(parts)
+
         # Pre-build reference strings
         references = ["\n\n## References\n\n"]
         references.extend(
-            f"⟨{num}⟩ {url}{desc}\n" 
+            f"⟨{num}⟩ {url}{desc}\n"
             for url, (num, desc) in sorted(link_map.items(), key=lambda x: x[1][0])
         )
-        
-        return converted_text, ''.join(references)
 
-    def generate_markdown(self, 
-                         cleaned_html: str, 
-                         base_url: str = "",
-                         html2text_options: Optional[Dict[str, Any]] = None,
-                         content_filter: Optional[RelevantContentFilter] = None,
-                         citations: bool = True,
-                         **kwargs) -> MarkdownGenerationResult:
-        """Generate markdown with citations from cleaned HTML."""
-        # Initialize HTML2Text with options
-        h = CustomHTML2Text()
-        if html2text_options:
-            h.update_params(**html2text_options)
+        return converted_text, "".join(references)
 
-        # Generate raw markdown
-        raw_markdown = h.handle(cleaned_html)
-        raw_markdown = raw_markdown.replace('    ```', '```')
+    def generate_markdown(
+        self,
+        cleaned_html: str,
+        base_url: str = "",
+        html2text_options: Optional[Dict[str, Any]] = None,
+        options: Optional[Dict[str, Any]] = None,
+        content_filter: Optional[RelevantContentFilter] = None,
+        citations: bool = True,
+        **kwargs,
+    ) -> MarkdownGenerationResult:
+        """
+        Generate markdown with citations from cleaned HTML.
 
-        # Convert links to citations
-        markdown_with_citations: str = ""
-        references_markdown: str = ""
-        if citations:
-            markdown_with_citations, references_markdown = self.convert_links_to_citations(
-                raw_markdown, base_url
+        How it works:
+        1. Generate raw markdown from cleaned HTML.
+        2. Convert links to citations.
+        3. Generate fit markdown if content filter is provided.
+        4. Return MarkdownGenerationResult.
+
+        Args:
+            cleaned_html (str): Cleaned HTML content.
+            base_url (str): Base URL for URL joins.
+            html2text_options (Optional[Dict[str, Any]]): HTML2Text options.
+            options (Optional[Dict[str, Any]]): Additional options for markdown generation.
+            content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
+            citations (bool): Whether to generate citations.
+
+        Returns:
+            MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
+        """
+        try:
+            # Initialize HTML2Text with default options for better conversion
+            h = CustomHTML2Text(baseurl=base_url)
+            default_options = {
+                "body_width": 0,  # Disable text wrapping
+                "ignore_emphasis": False,
+                "ignore_links": False,
+                "ignore_images": False,
+                "protect_links": False,
+                "single_line_break": True,
+                "mark_code": True,
+                "escape_snob": False,
+            }
+
+            # Update with custom options if provided
+            if html2text_options:
+                default_options.update(html2text_options)
+            elif options:
+                default_options.update(options)
+            elif self.options:
+                default_options.update(self.options)
+
+            h.update_params(**default_options)
+
+            # Ensure we have valid input
+            if not cleaned_html:
+                cleaned_html = ""
+            elif not isinstance(cleaned_html, str):
+                cleaned_html = str(cleaned_html)
+
+            # Generate raw markdown
+            try:
+                raw_markdown = h.handle(cleaned_html)
+            except Exception as e:
+                raw_markdown = f"Error converting HTML to markdown: {str(e)}"
+
+            raw_markdown = raw_markdown.replace("    ```", "```")
+
+            # Convert links to citations
+            markdown_with_citations: str = raw_markdown
+            references_markdown: str = ""
+            if citations:
+                try:
+                    (
+                        markdown_with_citations,
+                        references_markdown,
+                    ) = self.convert_links_to_citations(raw_markdown, base_url)
+                except Exception as e:
+                    markdown_with_citations = raw_markdown
+                    references_markdown = f"Error generating citations: {str(e)}"
+
+            # Generate fit markdown if content filter is provided
+            fit_markdown: Optional[str] = ""
+            filtered_html: Optional[str] = ""
+            if content_filter or self.content_filter:
+                try:
+                    content_filter = content_filter or self.content_filter
+                    filtered_html = content_filter.filter_content(cleaned_html)
+                    filtered_html = "\n".join(
+                        "<div>{}</div>".format(s) for s in filtered_html
+                    )
+                    fit_markdown = h.handle(filtered_html)
+                except Exception as e:
+                    fit_markdown = f"Error generating fit markdown: {str(e)}"
+                    filtered_html = ""
+
+            return MarkdownGenerationResult(
+                raw_markdown=raw_markdown or "",
+                markdown_with_citations=markdown_with_citations or "",
+                references_markdown=references_markdown or "",
+                fit_markdown=fit_markdown or "",
+                fit_html=filtered_html or "",
+            )
+        except Exception as e:
+            # If anything fails, return empty strings with error message
+            error_msg = f"Error in markdown generation: {str(e)}"
+            return MarkdownGenerationResult(
+                raw_markdown=error_msg,
+                markdown_with_citations=error_msg,
+                references_markdown="",
+                fit_markdown="",
+                fit_html="",
             )
-
-        # Generate fit markdown if content filter is provided
-        fit_markdown: Optional[str] = ""
-        filtered_html: Optional[str] = ""
-        if content_filter or self.content_filter:
-            content_filter = content_filter or self.content_filter
-            filtered_html = content_filter.filter_content(cleaned_html)
-            filtered_html = '\n'.join('<div>{}</div>'.format(s) for s in filtered_html)
-            fit_markdown = h.handle(filtered_html)
-
-        return MarkdownGenerationResult(
-            raw_markdown=raw_markdown,
-            markdown_with_citations=markdown_with_citations,
-            references_markdown=references_markdown,
-            fit_markdown=fit_markdown,
-            fit_html=filtered_html,
-        )
-
-def fast_urljoin(base: str, url: str) -> str:
-    """Fast URL joining for common cases."""
-    if url.startswith(('http://', 'https://', 'mailto:', '//')):
-        return url
-    if url.startswith('/'):
-        # Handle absolute paths
-        if base.endswith('/'):
-            return base[:-1] + url
-        return base + url
-    return urljoin(base, url)

crawl4ai/migrations.py

@@ -1,30 +1,33 @@
 import os
 import asyncio
-import logging
 from pathlib import Path
 import aiosqlite
 from typing import Optional
 import xxhash
 import aiofiles
 import shutil
-import time
 from datetime import datetime
+from .async_logger import AsyncLogger, LogLevel
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+# logging.basicConfig(level=logging.INFO)
+# logger = logging.getLogger(__name__)
 
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
 
 class DatabaseMigration:
     def __init__(self, db_path: str):
         self.db_path = db_path
         self.content_paths = self._ensure_content_dirs(os.path.dirname(db_path))
-        
+
     def _ensure_content_dirs(self, base_path: str) -> dict:
         dirs = {
-            'html': 'html_content',
-            'cleaned': 'cleaned_html',
-            'markdown': 'markdown_content', 
-            'extracted': 'extracted_content',
-            'screenshots': 'screenshots'
+            "html": "html_content",
+            "cleaned": "cleaned_html",
+            "markdown": "markdown_content",
+            "extracted": "extracted_content",
+            "screenshots": "screenshots",
         }
         content_paths = {}
         for key, dirname in dirs.items():
@@ -43,42 +46,55 @@ class DatabaseMigration:
     async def _store_content(self, content: str, content_type: str) -> str:
         if not content:
             return ""
-        
+
         content_hash = self._generate_content_hash(content)
         file_path = os.path.join(self.content_paths[content_type], content_hash)
-        
+
         if not os.path.exists(file_path):
-            async with aiofiles.open(file_path, 'w', encoding='utf-8') as f:
+            async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
                 await f.write(content)
-                
+
         return content_hash
 
     async def migrate_database(self):
         """Migrate existing database to file-based storage"""
-        logger.info("Starting database migration...")
-        
+        # logger.info("Starting database migration...")
+        logger.info("Starting database migration...", tag="INIT")
+
         try:
             async with aiosqlite.connect(self.db_path) as db:
                 # Get all rows
                 async with db.execute(
-                    '''SELECT url, html, cleaned_html, markdown, 
-                       extracted_content, screenshot FROM crawled_data'''
+                    """SELECT url, html, cleaned_html, markdown, 
+                       extracted_content, screenshot FROM crawled_data"""
                 ) as cursor:
                     rows = await cursor.fetchall()
 
                 migrated_count = 0
                 for row in rows:
-                    url, html, cleaned_html, markdown, extracted_content, screenshot = row
-                    
+                    (
+                        url,
+                        html,
+                        cleaned_html,
+                        markdown,
+                        extracted_content,
+                        screenshot,
+                    ) = row
+
                     # Store content in files and get hashes
-                    html_hash = await self._store_content(html, 'html')
-                    cleaned_hash = await self._store_content(cleaned_html, 'cleaned')
-                    markdown_hash = await self._store_content(markdown, 'markdown')
-                    extracted_hash = await self._store_content(extracted_content, 'extracted')
-                    screenshot_hash = await self._store_content(screenshot, 'screenshots')
+                    html_hash = await self._store_content(html, "html")
+                    cleaned_hash = await self._store_content(cleaned_html, "cleaned")
+                    markdown_hash = await self._store_content(markdown, "markdown")
+                    extracted_hash = await self._store_content(
+                        extracted_content, "extracted"
+                    )
+                    screenshot_hash = await self._store_content(
+                        screenshot, "screenshots"
+                    )
 
                     # Update database with hashes
-                    await db.execute('''
+                    await db.execute(
+                        """
                         UPDATE crawled_data 
                         SET html = ?, 
                             cleaned_html = ?,
@@ -86,67 +102,93 @@ class DatabaseMigration:
                             extracted_content = ?,
                             screenshot = ?
                         WHERE url = ?
-                    ''', (html_hash, cleaned_hash, markdown_hash, 
-                         extracted_hash, screenshot_hash, url))
-                    
+                    """,
+                        (
+                            html_hash,
+                            cleaned_hash,
+                            markdown_hash,
+                            extracted_hash,
+                            screenshot_hash,
+                            url,
+                        ),
+                    )
+
                     migrated_count += 1
                     if migrated_count % 100 == 0:
-                        logger.info(f"Migrated {migrated_count} records...")
+                        logger.info(f"Migrated {migrated_count} records...", tag="INIT")
 
                 await db.commit()
-                logger.info(f"Migration completed. {migrated_count} records processed.")
+                logger.success(
+                    f"Migration completed. {migrated_count} records processed.",
+                    tag="COMPLETE",
+                )
 
         except Exception as e:
-            logger.error(f"Migration failed: {e}")
-            raise
+            # logger.error(f"Migration failed: {e}")
+            logger.error(
+                message="Migration failed: {error}",
+                tag="ERROR",
+                params={"error": str(e)},
+            )
+            raise e
+
 
 async def backup_database(db_path: str) -> str:
     """Create backup of existing database"""
     if not os.path.exists(db_path):
-        logger.info("No existing database found. Skipping backup.")
+        logger.info("No existing database found. Skipping backup.", tag="INIT")
         return None
-        
+
     # Create backup with timestamp
-    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     backup_path = f"{db_path}.backup_{timestamp}"
-    
+
     try:
         # Wait for any potential write operations to finish
         await asyncio.sleep(1)
-        
+
         # Create backup
         shutil.copy2(db_path, backup_path)
-        logger.info(f"Database backup created at: {backup_path}")
+        logger.info(f"Database backup created at: {backup_path}", tag="COMPLETE")
         return backup_path
     except Exception as e:
-        logger.error(f"Backup failed: {e}")
-        raise
-    
+        # logger.error(f"Backup failed: {e}")
+        logger.error(
+            message="Migration failed: {error}", tag="ERROR", params={"error": str(e)}
+        )
+        raise e
+
+
 async def run_migration(db_path: Optional[str] = None):
     """Run database migration"""
     if db_path is None:
         db_path = os.path.join(Path.home(), ".crawl4ai", "crawl4ai.db")
-    
+
     if not os.path.exists(db_path):
-        logger.info("No existing database found. Skipping migration.")
+        logger.info("No existing database found. Skipping migration.", tag="INIT")
         return
-        
+
     # Create backup first
     backup_path = await backup_database(db_path)
     if not backup_path:
         return
-    
+
     migration = DatabaseMigration(db_path)
     await migration.migrate_database()
-    
+
+
 def main():
     """CLI entry point for migration"""
     import argparse
-    parser = argparse.ArgumentParser(description='Migrate Crawl4AI database to file-based storage')
-    parser.add_argument('--db-path', help='Custom database path')
+
+    parser = argparse.ArgumentParser(
+        description="Migrate Crawl4AI database to file-based storage"
+    )
+    parser.add_argument("--db-path", help="Custom database path")
     args = parser.parse_args()
-    
+
     asyncio.run(run_migration(args.db_path))
 
+
 if __name__ == "__main__":
-    main()
+    main()

crawl4ai/model_loader.py

@@ -2,109 +2,125 @@ from functools import lru_cache
 from pathlib import Path
 import subprocess, os
 import shutil
-import tarfile
 from .model_loader import *
 import argparse
-import urllib.request
 from crawl4ai.config import MODEL_REPO_BRANCH
+
 __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
 
+
 @lru_cache()
 def get_available_memory(device):
     import torch
-    if device.type == 'cuda':
+
+    if device.type == "cuda":
         return torch.cuda.get_device_properties(device).total_memory
-    elif device.type == 'mps':      
-        return 48 * 1024 ** 3  # Assuming 8GB for MPS, as a conservative estimate
+    elif device.type == "mps":
+        return 48 * 1024**3  # Assuming 8GB for MPS, as a conservative estimate
     else:
         return 0
 
+
 @lru_cache()
 def calculate_batch_size(device):
     available_memory = get_available_memory(device)
-    
-    if device.type == 'cpu':
+
+    if device.type == "cpu":
         return 16
-    elif device.type in ['cuda', 'mps']:
+    elif device.type in ["cuda", "mps"]:
         # Adjust these thresholds based on your model size and available memory
-        if available_memory >= 31 * 1024 ** 3:  # > 32GB
+        if available_memory >= 31 * 1024**3:  # > 32GB
             return 256
-        elif available_memory >= 15 * 1024 ** 3:  # > 16GB to 32GB
+        elif available_memory >= 15 * 1024**3:  # > 16GB to 32GB
             return 128
-        elif available_memory >= 8 * 1024 ** 3:  # 8GB to 16GB
+        elif available_memory >= 8 * 1024**3:  # 8GB to 16GB
             return 64
         else:
             return 32
     else:
-        return 16  # Default batch size   
-    
+        return 16  # Default batch size
+
+
 @lru_cache()
 def get_device():
     import torch
+
     if torch.cuda.is_available():
-        device = torch.device('cuda')
+        device = torch.device("cuda")
     elif torch.backends.mps.is_available():
-        device = torch.device('mps')
+        device = torch.device("mps")
     else:
-        device = torch.device('cpu')
-    return device   
-    
+        device = torch.device("cpu")
+    return device
+
+
 def set_model_device(model):
     device = get_device()
-    model.to(device)    
+    model.to(device)
     return model, device
 
+
 @lru_cache()
 def get_home_folder():
-    home_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+    home_folder = os.path.join(
+        os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
+    )
     os.makedirs(home_folder, exist_ok=True)
     os.makedirs(f"{home_folder}/cache", exist_ok=True)
     os.makedirs(f"{home_folder}/models", exist_ok=True)
-    return home_folder 
+    return home_folder
+
 
 @lru_cache()
 def load_bert_base_uncased():
-    from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
-    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', resume_download=None)
-    model = BertModel.from_pretrained('bert-base-uncased', resume_download=None)
+    from transformers import BertTokenizer, BertModel
+
+    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", resume_download=None)
+    model = BertModel.from_pretrained("bert-base-uncased", resume_download=None)
     model.eval()
     model, device = set_model_device(model)
     return tokenizer, model
 
+
 @lru_cache()
 def load_HF_embedding_model(model_name="BAAI/bge-small-en-v1.5") -> tuple:
     """Load the Hugging Face model for embedding.
-    
+
     Args:
         model_name (str, optional): The model name to load. Defaults to "BAAI/bge-small-en-v1.5".
-        
+
     Returns:
         tuple: The tokenizer and model.
     """
-    from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
+    from transformers import AutoTokenizer, AutoModel
+
     tokenizer = AutoTokenizer.from_pretrained(model_name, resume_download=None)
     model = AutoModel.from_pretrained(model_name, resume_download=None)
     model.eval()
     model, device = set_model_device(model)
     return tokenizer, model
 
+
 @lru_cache()
 def load_text_classifier():
     from transformers import AutoTokenizer, AutoModelForSequenceClassification
     from transformers import pipeline
-    import torch
 
-    tokenizer = AutoTokenizer.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
-    model = AutoModelForSequenceClassification.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
+    tokenizer = AutoTokenizer.from_pretrained(
+        "dstefa/roberta-base_topic_classification_nyt_news"
+    )
+    model = AutoModelForSequenceClassification.from_pretrained(
+        "dstefa/roberta-base_topic_classification_nyt_news"
+    )
     model.eval()
     model, device = set_model_device(model)
     pipe = pipeline("text-classification", model=model, tokenizer=tokenizer)
     return pipe
 
+
 @lru_cache()
 def load_text_multilabel_classifier():
     from transformers import AutoModelForSequenceClassification, AutoTokenizer
-    import numpy as np
     from scipy.special import expit
     import torch
 
@@ -116,18 +132,27 @@ def load_text_multilabel_classifier():
     # else:
     #     device = torch.device("cpu")
     #     # return load_spacy_model(), torch.device("cpu")
-    
 
     MODEL = "cardiffnlp/tweet-topic-21-multi"
     tokenizer = AutoTokenizer.from_pretrained(MODEL, resume_download=None)
-    model = AutoModelForSequenceClassification.from_pretrained(MODEL, resume_download=None)
+    model = AutoModelForSequenceClassification.from_pretrained(
+        MODEL, resume_download=None
+    )
     model.eval()
     model, device = set_model_device(model)
     class_mapping = model.config.id2label
 
     def _classifier(texts, threshold=0.5, max_length=64):
-        tokens = tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=max_length)
-        tokens = {key: val.to(device) for key, val in tokens.items()}  # Move tokens to the selected device
+        tokens = tokenizer(
+            texts,
+            return_tensors="pt",
+            padding=True,
+            truncation=True,
+            max_length=max_length,
+        )
+        tokens = {
+            key: val.to(device) for key, val in tokens.items()
+        }  # Move tokens to the selected device
 
         with torch.no_grad():
             output = model(**tokens)
@@ -138,35 +163,41 @@ def load_text_multilabel_classifier():
 
         batch_labels = []
         for prediction in predictions:
-            labels = [class_mapping[i] for i, value in enumerate(prediction) if value == 1]
+            labels = [
+                class_mapping[i] for i, value in enumerate(prediction) if value == 1
+            ]
             batch_labels.append(labels)
 
         return batch_labels
 
     return _classifier, device
 
+
 @lru_cache()
 def load_nltk_punkt():
     import nltk
+
     try:
-        nltk.data.find('tokenizers/punkt')
+        nltk.data.find("tokenizers/punkt")
     except LookupError:
-        nltk.download('punkt')
-    return nltk.data.find('tokenizers/punkt')
+        nltk.download("punkt")
+    return nltk.data.find("tokenizers/punkt")
+
 
 @lru_cache()
 def load_spacy_model():
     import spacy
+
     name = "models/reuters"
     home_folder = get_home_folder()
     model_folder = Path(home_folder) / name
-    
+
     # Check if the model directory already exists
     if not (model_folder.exists() and any(model_folder.iterdir())):
         repo_url = "https://github.com/unclecode/crawl4ai.git"
-        branch = MODEL_REPO_BRANCH 
+        branch = MODEL_REPO_BRANCH
         repo_folder = Path(home_folder) / "crawl4ai"
-        
+
         print("[LOG] ⏬ Downloading Spacy model for the first time...")
 
         # Remove existing repo folder if it exists
@@ -176,7 +207,9 @@ def load_spacy_model():
                 if model_folder.exists():
                     shutil.rmtree(model_folder)
             except PermissionError:
-                print("[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:")
+                print(
+                    "[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:"
+                )
                 print(f"- {repo_folder}")
                 print(f"- {model_folder}")
                 return None
@@ -187,7 +220,7 @@ def load_spacy_model():
                 ["git", "clone", "-b", branch, repo_url, str(repo_folder)],
                 stdout=subprocess.DEVNULL,
                 stderr=subprocess.DEVNULL,
-                check=True
+                check=True,
             )
 
             # Create the models directory if it doesn't exist
@@ -215,6 +248,7 @@ def load_spacy_model():
         print(f"Error loading spacy model: {e}")
         return None
 
+
 def download_all_models(remove_existing=False):
     """Download all models required for Crawl4AI."""
     if remove_existing:
@@ -243,14 +277,20 @@ def download_all_models(remove_existing=False):
     load_nltk_punkt()
     print("[LOG] ✅ All models downloaded successfully.")
 
+
 def main():
     print("[LOG] Welcome to the Crawl4AI Model Downloader!")
     print("[LOG] This script will download all the models required for Crawl4AI.")
     parser = argparse.ArgumentParser(description="Crawl4AI Model Downloader")
-    parser.add_argument('--remove-existing', action='store_true', help="Remove existing models before downloading")
+    parser.add_argument(
+        "--remove-existing",
+        action="store_true",
+        help="Remove existing models before downloading",
+    )
     args = parser.parse_args()
-    
+
     download_all_models(remove_existing=args.remove_existing)
 
+
 if __name__ == "__main__":
     main()

crawl4ai/models.py

@@ -1,12 +1,84 @@
-from pydantic import BaseModel, HttpUrl
-from typing import List, Dict, Optional, Callable, Awaitable, Union
+from re import U
+from pydantic import BaseModel, HttpUrl, PrivateAttr
+from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
+from enum import Enum
+from dataclasses import dataclass
+from .ssl_certificate import SSLCertificate
+from datetime import datetime
+from datetime import timedelta
 
 
+###############################
+# Dispatcher Models
+###############################
+@dataclass
+class DomainState:
+    last_request_time: float = 0
+    current_delay: float = 0
+    fail_count: int = 0
+
+
+@dataclass
+class CrawlerTaskResult:
+    task_id: str
+    url: str
+    result: "CrawlResult"
+    memory_usage: float
+    peak_memory: float
+    start_time: Union[datetime, float]
+    end_time: Union[datetime, float]
+    error_message: str = ""
+
+
+class CrawlStatus(Enum):
+    QUEUED = "QUEUED"
+    IN_PROGRESS = "IN_PROGRESS"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+
+
+@dataclass
+class CrawlStats:
+    task_id: str
+    url: str
+    status: CrawlStatus
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+    memory_usage: float = 0.0
+    peak_memory: float = 0.0
+    error_message: str = ""
+
+    @property
+    def duration(self) -> str:
+        if not self.start_time:
+            return "0:00"
+        end = self.end_time or datetime.now()
+        duration = end - self.start_time
+        return str(timedelta(seconds=int(duration.total_seconds())))
+
+
+class DisplayMode(Enum):
+    DETAILED = "DETAILED"
+    AGGREGATED = "AGGREGATED"
+
+
+###############################
+# Crawler Models
+###############################
+@dataclass
+class TokenUsage:
+    completion_tokens: int = 0
+    prompt_tokens: int = 0
+    total_tokens: int = 0
+    completion_tokens_details: Optional[dict] = None
+    prompt_tokens_details: Optional[dict] = None
+
 
 class UrlModel(BaseModel):
     url: HttpUrl
     forced: bool = False
 
+
 class MarkdownGenerationResult(BaseModel):
     raw_markdown: str
     markdown_with_citations: str
@@ -14,6 +86,28 @@ class MarkdownGenerationResult(BaseModel):
     fit_markdown: Optional[str] = None
     fit_html: Optional[str] = None
 
+    def __str__(self):
+        return self.raw_markdown
+
+@dataclass
+class TraversalStats:
+    """Statistics for the traversal process"""
+
+    start_time: datetime = datetime.now()
+    urls_processed: int = 0
+    urls_failed: int = 0
+    urls_skipped: int = 0
+    total_depth_reached: int = 0
+    current_depth: int = 0
+
+class DispatchResult(BaseModel):
+    task_id: str
+    memory_usage: float
+    peak_memory: float
+    start_time: Union[datetime, float]
+    end_time: Union[datetime, float]
+    error_message: str = ""
+
 class CrawlResult(BaseModel):
     url: str
     html: str
@@ -22,27 +116,192 @@ class CrawlResult(BaseModel):
     media: Dict[str, List[Dict]] = {}
     links: Dict[str, List[Dict]] = {}
     downloaded_files: Optional[List[str]] = None
+    js_execution_result: Optional[Dict[str, Any]] = None
     screenshot: Optional[str] = None
-    markdown: Optional[Union[str, MarkdownGenerationResult]] = None
-    markdown_v2: Optional[MarkdownGenerationResult] = None
-    fit_markdown: Optional[str] = None
-    fit_html: Optional[str] = None
+    pdf: Optional[bytes] = None
+    _markdown: Optional[MarkdownGenerationResult] = PrivateAttr(default=None)
     extracted_content: Optional[str] = None
     metadata: Optional[dict] = None
     error_message: Optional[str] = None
     session_id: Optional[str] = None
     response_headers: Optional[dict] = None
     status_code: Optional[int] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+    dispatch_result: Optional[DispatchResult] = None
+    redirected_url: Optional[str] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+
+# NOTE: The StringCompatibleMarkdown class, custom __init__ method, property getters/setters,
+# and model_dump override all exist to support a smooth transition from markdown as a string
+# to markdown as a MarkdownGenerationResult object, while maintaining backward compatibility.
+# 
+# This allows code that expects markdown to be a string to continue working, while also
+# providing access to the full MarkdownGenerationResult object's properties.
+# 
+# The markdown_v2 property is deprecated and raises an error directing users to use markdown.
+# 
+# When backward compatibility is no longer needed in future versions, this entire mechanism
+# can be simplified to a standard field with no custom accessors or serialization logic.
     
+    def __init__(self, **data):
+        markdown_result = data.pop('markdown', None)
+        super().__init__(**data)
+        if markdown_result is not None:
+            self._markdown = markdown_result
+    
+    @property
+    def markdown(self):
+        """
+        Property that returns a StringCompatibleMarkdown object that behaves like
+        a string but also provides access to MarkdownGenerationResult attributes.
+        
+        This approach allows backward compatibility with code that expects 'markdown'
+        to be a string, while providing access to the full MarkdownGenerationResult.
+        """
+        if self._markdown is None:
+            return None
+        return StringCompatibleMarkdown(self._markdown)
+    
+    @markdown.setter
+    def markdown(self, value):
+        """
+        Setter for the markdown property.
+        """
+        self._markdown = value
+    
+    @property
+    def markdown_v2(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+
+        This property exists to inform users that 'markdown_v2' has been
+        deprecated and they should use 'markdown' instead.
+        """
+        raise AttributeError(
+            "The 'markdown_v2' attribute is deprecated and has been removed. "
+            """Please use 'markdown' instead, which now returns a MarkdownGenerationResult, with
+            following properties:
+            - raw_markdown: The raw markdown string
+            - markdown_with_citations: The markdown string with citations
+            - references_markdown: The markdown string with references
+            - fit_markdown: The markdown string with fit text
+            """
+        )
+    
+    @property
+    def fit_markdown(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+        """
+        raise AttributeError(
+            "The 'fit_markdown' attribute is deprecated and has been removed. "
+            "Please use 'markdown.fit_markdown' instead."
+        )
+    
+    @property
+    def fit_html(self):
+        """
+        Deprecated property that raises an AttributeError when accessed.
+        """
+        raise AttributeError(
+            "The 'fit_html' attribute is deprecated and has been removed. "
+            "Please use 'markdown.fit_html' instead."
+        )
+
+    def model_dump(self, *args, **kwargs):
+        """
+        Override model_dump to include the _markdown private attribute in serialization.
+        
+        This override is necessary because:
+        1. PrivateAttr fields are excluded from serialization by default
+        2. We need to maintain backward compatibility by including the 'markdown' field
+           in the serialized output
+        3. We're transitioning from 'markdown_v2' to enhancing 'markdown' to hold
+           the same type of data
+        
+        Future developers: This method ensures that the markdown content is properly
+        serialized despite being stored in a private attribute. If the serialization
+        requirements change, this is where you would update the logic.
+        """
+        result = super().model_dump(*args, **kwargs)
+        if self._markdown is not None:
+            result["markdown"] = self._markdown.model_dump() 
+        return result
+
+class StringCompatibleMarkdown(str):
+    """A string subclass that also provides access to MarkdownGenerationResult attributes"""
+    def __new__(cls, markdown_result):
+        return super().__new__(cls, markdown_result.raw_markdown)
+    
+    def __init__(self, markdown_result):
+        self._markdown_result = markdown_result
+    
+    def __getattr__(self, name):
+        return getattr(self._markdown_result, name)
+
+# END of backward compatibility code for markdown/markdown_v2.
+# When removing this code in the future, make sure to:
+# 1. Replace the private attribute and property with a standard field
+# 2. Update any serialization logic that might depend on the current behavior
+
 class AsyncCrawlResponse(BaseModel):
     html: str
     response_headers: Dict[str, str]
+    js_execution_result: Optional[Dict[str, Any]] = None
     status_code: int
     screenshot: Optional[str] = None
+    pdf_data: Optional[bytes] = None
     get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
     downloaded_files: Optional[List[str]] = None
+    ssl_certificate: Optional[SSLCertificate] = None
+    redirected_url: Optional[str] = None
 
     class Config:
         arbitrary_types_allowed = True
 
 
+###############################
+# Scraping Models
+###############################
+class MediaItem(BaseModel):
+    src: Optional[str] = ""
+    data: Optional[str] = ""
+    alt: Optional[str] = ""
+    desc: Optional[str] = ""
+    score: Optional[int] = 0
+    type: str = "image"
+    group_id: Optional[int] = 0
+    format: Optional[str] = None
+    width: Optional[int] = None
+
+
+class Link(BaseModel):
+    href: Optional[str] = ""
+    text: Optional[str] = ""
+    title: Optional[str] = ""
+    base_domain: Optional[str] = ""
+
+
+class Media(BaseModel):
+    images: List[MediaItem] = []
+    videos: List[
+        MediaItem
+    ] = []  # Using MediaItem model for now, can be extended with Video model if needed
+    audios: List[
+        MediaItem
+    ] = []  # Using MediaItem model for now, can be extended with Audio model if needed
+
+
+class Links(BaseModel):
+    internal: List[Link] = []
+    external: List[Link] = []
+
+
+class ScrapingResult(BaseModel):
+    cleaned_html: str
+    success: bool
+    media: Media = Media()
+    links: Links = Links()
+    metadata: Dict[str, Any] = {}

crawl4ai/processors/pdf/__init__.py

@@ -0,0 +1,165 @@
+from pathlib import Path
+import asyncio
+from dataclasses import asdict
+from crawl4ai.async_logger import AsyncLogger
+from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy
+from crawl4ai.models import AsyncCrawlResponse, ScrapingResult 
+from crawl4ai.content_scraping_strategy import ContentScrapingStrategy
+from .processor import NaivePDFProcessorStrategy  # Assuming your current PDF code is in pdf_processor.py
+
+class PDFCrawlerStrategy(AsyncCrawlerStrategy):
+    def __init__(self, logger: AsyncLogger = None):
+        self.logger = logger
+        
+    async def crawl(self, url: str, **kwargs) -> AsyncCrawlResponse:
+        # Just pass through with empty HTML - scraper will handle actual processing
+        return AsyncCrawlResponse(
+            html="",  # Scraper will handle the real work
+            response_headers={"Content-Type": "application/pdf"},
+            status_code=200
+        )
+    
+    async def close(self):
+        pass        
+        
+    async def __aenter__(self):        
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+
+class PDFContentScrapingStrategy(ContentScrapingStrategy):
+    """
+    A content scraping strategy for PDF files.
+    
+    Attributes:
+        save_images_locally (bool): Whether to save images locally.
+        extract_images (bool): Whether to extract images from PDF.
+        image_save_dir (str): Directory to save extracted images.
+        logger (AsyncLogger): Logger instance for recording events and errors.
+        
+    Methods:
+        scrap(url: str, html: str, **params) -> ScrapingResult:
+            Scrap content from a PDF file.
+        ascrap(url: str, html: str, **kwargs) -> ScrapingResult:
+            Asynchronous version of scrap.
+            
+    Usage:
+        strategy = PDFContentScrapingStrategy(
+            save_images_locally=False,
+            extract_images=False,
+            image_save_dir=None,
+            logger=logger
+        )
+        
+    """
+    def __init__(self, 
+                 save_images_locally : bool = False,
+                 extract_images : bool = False,
+                 image_save_dir : str = None,
+                 batch_size: int = 4,
+                 logger: AsyncLogger = None):
+        self.logger = logger
+        self.pdf_processor = NaivePDFProcessorStrategy(
+            save_images_locally=save_images_locally,
+            extract_images=extract_images,
+            image_save_dir=image_save_dir,
+            batch_size=batch_size
+        )
+
+    def scrap(self, url: str, html: str, **params) -> ScrapingResult:
+        """
+        Scrap content from a PDF file.
+        
+        Args:
+            url (str): The URL of the PDF file.
+            html (str): The HTML content of the page.
+            **params: Additional parameters.
+        
+        Returns:
+            ScrapingResult: The scraped content.
+        """
+        # Download if URL or use local path
+        pdf_path = self._get_pdf_path(url)
+        try:
+            # Process PDF
+            # result = self.pdf_processor.process(Path(pdf_path))
+            result = self.pdf_processor.process_batch(Path(pdf_path))
+            
+            # Combine page HTML
+            cleaned_html = f"""
+        <html>
+            <head><meta name="pdf-pages" content="{len(result.pages)}"></head>
+            <body>
+                {''.join(f'<div class="pdf-page" data-page="{i+1}">{page.html}</div>'
+                         for i, page in enumerate(result.pages))}
+            </body>
+        </html>
+        """
+            
+            # Accumulate media and links with page numbers
+            media = {"images": []}
+            links = {"urls": []}
+            
+            for page in result.pages:
+                # Add page number to each image
+                for img in page.images:
+                    img["page"] = page.page_number
+                    media["images"].append(img)
+                
+                # Add page number to each link
+                for link in page.links:
+                    links["urls"].append({
+                        "url": link,
+                        "page": page.page_number
+                    })
+
+            return ScrapingResult(
+                cleaned_html=cleaned_html,
+                success=True,
+                media=media,
+                links=links,
+                metadata=asdict(result.metadata)
+            )
+        finally:
+            # Cleanup temp file if downloaded
+            if url.startswith(("http://", "https://")):
+                Path(pdf_path).unlink(missing_ok=True)
+
+    async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
+        # For simple cases, you can use the sync version
+        return await asyncio.to_thread(self.scrap, url, html, **kwargs)
+        
+
+    def _get_pdf_path(self, url: str) -> str:
+        if url.startswith(("http://", "https://")):
+            import tempfile
+            import requests
+            
+            # Create temp file with .pdf extension
+            temp_file = tempfile.NamedTemporaryFile(suffix='.pdf', delete=False)
+            
+            try:
+                # Download PDF with streaming
+                response = requests.get(url, stream=True)
+                response.raise_for_status()
+                
+                # Write to temp file
+                with open(temp_file.name, 'wb') as f:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        f.write(chunk)
+                        
+                return temp_file.name
+                
+            except Exception as e:
+                # Clean up temp file if download fails
+                Path(temp_file.name).unlink(missing_ok=True)
+                raise RuntimeError(f"Failed to download PDF from {url}: {str(e)}")
+                
+        elif url.startswith("file://"):
+            return url[7:]  # Strip file:// prefix
+            
+        return url  # Assume local path
+    
+
+__all__ = ["PDFCrawlerStrategy", "PDFContentScrapingStrategy"]

crawl4ai/processors/pdf/processor.py

@@ -0,0 +1,487 @@
+import logging
+import re
+from abc import ABC, abstractmethod
+from datetime import datetime
+from pathlib import Path
+from time import time
+from dataclasses import dataclass, asdict, field
+from typing import Dict, List, Optional, Any, Union
+import base64
+import tempfile
+from .utils import *
+from .utils import (
+    apply_png_predictor,
+    clean_pdf_text,
+    clean_pdf_text_to_html,
+)
+
+# Remove direct PyPDF2 imports from the top
+# import PyPDF2
+# from PyPDF2 import PdfReader
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class PDFMetadata:
+    title: Optional[str] = None
+    author: Optional[str] = None
+    producer: Optional[str] = None
+    created: Optional[datetime] = None
+    modified: Optional[datetime] = None
+    pages: int = 0
+    encrypted: bool = False
+    file_size: Optional[int] = None
+
+@dataclass
+class PDFPage:
+    page_number: int
+    raw_text: str = ""
+    markdown: str = ""
+    html: str = ""
+    images: List[Dict] = field(default_factory=list)
+    links: List[str] = field(default_factory=list)
+    layout: List[Dict] = field(default_factory=list)
+
+@dataclass
+class PDFProcessResult:
+    metadata: PDFMetadata
+    pages: List[PDFPage]
+    processing_time: float = 0.0
+    version: str = "1.0"
+
+class PDFProcessorStrategy(ABC):
+    @abstractmethod
+    def process(self, pdf_path: Path) -> PDFProcessResult:
+        pass
+
+class NaivePDFProcessorStrategy(PDFProcessorStrategy):
+    def __init__(self, image_dpi: int = 144, image_quality: int = 85, extract_images: bool = True, 
+                 save_images_locally: bool = False, image_save_dir: Optional[Path] = None, batch_size: int = 4):
+        # Import check at initialization time
+        try:
+            import PyPDF2
+        except ImportError:
+            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            
+        self.image_dpi = image_dpi
+        self.image_quality = image_quality
+        self.current_page_number = 0
+        self.extract_images = extract_images
+        self.save_images_locally = save_images_locally
+        self.image_save_dir = image_save_dir
+        self.batch_size = batch_size
+        self._temp_dir = None
+
+    def process(self, pdf_path: Path) -> PDFProcessResult:
+        # Import inside method to allow dependency to be optional
+        try:
+            from PyPDF2 import PdfReader
+        except ImportError:
+            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            
+        start_time = time()
+        result = PDFProcessResult(
+            metadata=PDFMetadata(),
+            pages=[],
+            version="1.1"
+        )
+
+        try:
+            with pdf_path.open('rb') as file:
+                reader = PdfReader(file)
+                result.metadata = self._extract_metadata(pdf_path, reader)
+                
+                # Handle image directory
+                image_dir = None
+                if self.extract_images and self.save_images_locally:
+                    if self.image_save_dir:
+                        image_dir = Path(self.image_save_dir)
+                        image_dir.mkdir(exist_ok=True, parents=True)
+                    else:
+                        self._temp_dir = tempfile.mkdtemp(prefix='pdf_images_')
+                        image_dir = Path(self._temp_dir)
+
+                for page_num, page in enumerate(reader.pages):
+                    self.current_page_number = page_num + 1
+                    pdf_page = self._process_page(page, image_dir)
+                    result.pages.append(pdf_page)
+
+        except Exception as e:
+            logger.error(f"Failed to process PDF: {str(e)}")
+            raise
+        finally:
+            # Cleanup temp directory if it was created
+            if self._temp_dir and not self.image_save_dir:
+                import shutil
+                try:
+                    shutil.rmtree(self._temp_dir)
+                except Exception as e:
+                    logger.error(f"Failed to cleanup temp directory: {str(e)}")
+
+        result.processing_time = time() - start_time
+        return result
+
+    def process_batch(self, pdf_path: Path) -> PDFProcessResult:
+        """Like process() but processes PDF pages in parallel batches"""
+        # Import inside method to allow dependency to be optional
+        try:
+            from PyPDF2 import PdfReader
+            import PyPDF2  # For type checking
+        except ImportError:
+            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            
+        import concurrent.futures
+        import threading
+        
+        # Initialize PyPDF2 thread support
+        if not hasattr(threading.current_thread(), "_children"): 
+            threading.current_thread()._children = set()
+        
+        start_time = time()
+        result = PDFProcessResult(
+            metadata=PDFMetadata(),
+            pages=[],
+            version="1.1" 
+        )
+
+        try:
+            # Get metadata and page count from main thread
+            with pdf_path.open('rb') as file:
+                reader = PdfReader(file)
+                result.metadata = self._extract_metadata(pdf_path, reader)
+                total_pages = len(reader.pages)
+
+            # Handle image directory setup
+            image_dir = None
+            if self.extract_images and self.save_images_locally:
+                if self.image_save_dir:
+                    image_dir = Path(self.image_save_dir)
+                    image_dir.mkdir(exist_ok=True, parents=True)
+                else:
+                    self._temp_dir = tempfile.mkdtemp(prefix='pdf_images_')
+                    image_dir = Path(self._temp_dir)
+
+            def process_page_safely(page_num: int):
+                # Each thread opens its own file handle
+                with pdf_path.open('rb') as file:
+                    thread_reader = PdfReader(file)
+                    page = thread_reader.pages[page_num]
+                    self.current_page_number = page_num + 1
+                    return self._process_page(page, image_dir)
+
+            # Process pages in parallel batches
+            with concurrent.futures.ThreadPoolExecutor(max_workers=self.batch_size) as executor:
+                futures = []
+                for page_num in range(total_pages):
+                    future = executor.submit(process_page_safely, page_num)
+                    futures.append((page_num + 1, future))
+
+                # Collect results in order
+                result.pages = [None] * total_pages
+                for page_num, future in futures:
+                    try:
+                        pdf_page = future.result()
+                        result.pages[page_num - 1] = pdf_page
+                    except Exception as e:
+                        logger.error(f"Failed to process page {page_num}: {str(e)}")
+                        raise
+
+        except Exception as e:
+            logger.error(f"Failed to process PDF: {str(e)}")
+            raise
+        finally:
+            # Cleanup temp directory if it was created
+            if self._temp_dir and not self.image_save_dir:
+                import shutil
+                try:
+                    shutil.rmtree(self._temp_dir)
+                except Exception as e:
+                    logger.error(f"Failed to cleanup temp directory: {str(e)}")
+
+        result.processing_time = time() - start_time
+        return result
+
+    def _process_page(self, page, image_dir: Optional[Path]) -> PDFPage:
+        pdf_page = PDFPage(
+            page_number=self.current_page_number,
+        )
+
+        # Text and font extraction
+        def visitor_text(text, cm, tm, font_dict, font_size):
+            pdf_page.raw_text += text
+            pdf_page.layout.append({
+                "type": "text",
+                "text": text,
+                "x": tm[4],
+                "y": tm[5],
+            })
+        
+        page.extract_text(visitor_text=visitor_text)
+
+        # Image extraction
+        if self.extract_images:
+            pdf_page.images = self._extract_images(page, image_dir)
+
+        # Link extraction
+        pdf_page.links = self._extract_links(page)
+        
+        # Add markdown content
+        pdf_page.markdown = clean_pdf_text(self.current_page_number, pdf_page.raw_text)
+        pdf_page.html = clean_pdf_text_to_html(self.current_page_number, pdf_page.raw_text)
+
+        return pdf_page
+
+    def _extract_images(self, page, image_dir: Optional[Path]) -> List[Dict]:
+        # Import PyPDF2 for type checking only when needed
+        try:
+            import PyPDF2
+        except ImportError:
+            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            
+        if not self.extract_images:
+            return []
+
+        images = []
+        try:
+            resources = page.get("/Resources")
+            if resources:  # Check if resources exist
+                resources = resources.get_object()  # Resolve IndirectObject
+                if '/XObject' in resources:
+                    xobjects = resources['/XObject'].get_object()
+                    img_count = 0
+                    for obj_name in xobjects:
+                        xobj = xobjects[obj_name]
+                        if hasattr(xobj, 'get_object') and callable(xobj.get_object):
+                            xobj = xobj.get_object()
+                            if xobj.get('/Subtype') == '/Image':
+                                try:
+                                    img_count += 1
+                                    img_filename = f"page_{self.current_page_number}_img_{img_count}"
+                                    data = xobj.get_data()
+                                    filters = xobj.get('/Filter', [])
+                                    if not isinstance(filters, list):
+                                        filters = [filters]
+
+                                    # Resolve IndirectObjects in properties
+                                    width = xobj.get('/Width', 0)
+                                    height = xobj.get('/Height', 0)
+                                    color_space = xobj.get('/ColorSpace', '/DeviceRGB')
+                                    if isinstance(color_space, PyPDF2.generic.IndirectObject):
+                                        color_space = color_space.get_object()
+
+                                    # Handle different image encodings
+                                    success = False
+                                    image_format = 'bin'
+                                    image_data = None
+                                    
+                                    if '/FlateDecode' in filters:
+                                        try:
+                                            decode_parms = xobj.get('/DecodeParms', {})
+                                            if isinstance(decode_parms, PyPDF2.generic.IndirectObject):
+                                                decode_parms = decode_parms.get_object()
+                                            
+                                            predictor = decode_parms.get('/Predictor', 1)
+                                            bits = xobj.get('/BitsPerComponent', 8)
+                                            colors = 3 if color_space == '/DeviceRGB' else 1
+
+                                            if predictor >= 10:
+                                                data = apply_png_predictor(data, width, bits, colors)
+
+                                            # Create PIL Image
+                                            from PIL import Image
+                                            mode = 'RGB' if color_space == '/DeviceRGB' else 'L'
+                                            img = Image.frombytes(mode, (width, height), data)
+                                            
+                                            if self.save_images_locally:
+                                                final_path = (image_dir / img_filename).with_suffix('.png')
+                                                img.save(final_path)
+                                                image_data = str(final_path)
+                                            else:
+                                                import io
+                                                img_byte_arr = io.BytesIO()
+                                                img.save(img_byte_arr, format='PNG')
+                                                image_data = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
+                                            
+                                            success = True
+                                            image_format = 'png'
+                                        except Exception as e:
+                                            logger.error(f"FlateDecode error: {str(e)}")
+
+                                    elif '/DCTDecode' in filters:
+                                        # JPEG image
+                                        try:
+                                            if self.save_images_locally:
+                                                final_path = (image_dir / img_filename).with_suffix('.jpg')
+                                                with open(final_path, 'wb') as f:
+                                                    f.write(data)
+                                                image_data = str(final_path)
+                                            else:
+                                                image_data = base64.b64encode(data).decode('utf-8')
+                                            success = True
+                                            image_format = 'jpeg'
+                                        except Exception as e:
+                                            logger.error(f"JPEG save error: {str(e)}")
+
+                                    elif '/CCITTFaxDecode' in filters:
+                                        try:
+                                            if data[:4] != b'II*\x00':
+                                                # Add TIFF header if missing
+                                                tiff_header = b'II*\x00\x08\x00\x00\x00\x0e\x00\x00\x01\x03\x00\x01\x00\x00\x00' + \
+                                                            width.to_bytes(4, 'little') + \
+                                                            b'\x01\x03\x00\x01\x00\x00\x00' + \
+                                                            height.to_bytes(4, 'little') + \
+                                                            b'\x01\x12\x00\x03\x00\x00\x00\x01\x00\x01\x00\x00\x01\x17\x00\x04\x00\x00\x00\x01\x00\x00\x00J\x01\x1B\x00\x05\x00\x00\x00\x01\x00\x00\x00R\x01\x28\x00\x03\x00\x00\x00\x01\x00\x02\x00\x00'
+                                                data = tiff_header + data
+                                            
+                                            if self.save_images_locally:
+                                                final_path = (image_dir / img_filename).with_suffix('.tiff')
+                                                with open(final_path, 'wb') as f:
+                                                    f.write(data)
+                                                image_data = str(final_path)
+                                            else:
+                                                image_data = base64.b64encode(data).decode('utf-8')
+                                            success = True
+                                            image_format = 'tiff'
+                                        except Exception as e:
+                                            logger.error(f"CCITT save error: {str(e)}")
+
+                                    elif '/JPXDecode' in filters:
+                                        # JPEG 2000
+                                        try:
+                                            if self.save_images_locally:
+                                                final_path = (image_dir / img_filename).with_suffix('.jp2')
+                                                with open(final_path, 'wb') as f:
+                                                    f.write(data)
+                                                image_data = str(final_path)
+                                            else:
+                                                image_data = base64.b64encode(data).decode('utf-8')
+                                            success = True
+                                            image_format = 'jpeg2000'
+                                        except Exception as e:
+                                            logger.error(f"JPEG2000 save error: {str(e)}")
+
+                                    if success and image_data:
+                                        image_info = {
+                                            "format": image_format,
+                                            "width": width,
+                                            "height": height,
+                                            "color_space": str(color_space),
+                                            "bits_per_component": xobj.get('/BitsPerComponent', 1)
+                                        }
+                                        
+                                        if self.save_images_locally:
+                                            image_info["path"] = image_data
+                                        else:
+                                            image_info["data"] = image_data
+                                            
+                                        images.append(image_info)
+                                    else:
+                                        # Fallback: Save raw data
+                                        if self.save_images_locally:
+                                            final_path = (image_dir / img_filename).with_suffix('.bin')
+                                            with open(final_path, 'wb') as f:
+                                                f.write(data)
+                                            logger.warning(f"Saved raw image data to {final_path}")
+                                        else:
+                                            image_data = base64.b64encode(data).decode('utf-8')
+                                            images.append({
+                                                "format": "bin",
+                                                "width": width,
+                                                "height": height,
+                                                "color_space": str(color_space),
+                                                "bits_per_component": xobj.get('/BitsPerComponent', 1),
+                                                "data": image_data
+                                            })
+
+                                except Exception as e:
+                                    logger.error(f"Error processing image: {str(e)}")
+        except Exception as e:
+            logger.error(f"Image extraction error: {str(e)}")
+        
+        return images
+
+    def _extract_links(self, page) -> List[str]:
+        links = []
+        if '/Annots' in page:
+            try:
+                for annot in page['/Annots']:
+                    a = annot.get_object()
+                    if '/A' in a and '/URI' in a['/A']:
+                        links.append(a['/A']['/URI'])
+            except Exception as e:
+                print(f"Link error: {str(e)}")
+        return links
+
+    def _extract_metadata(self, pdf_path: Path, reader = None) -> PDFMetadata:
+        # Import inside method to allow dependency to be optional 
+        if reader is None:
+            try:
+                from PyPDF2 import PdfReader
+                reader = PdfReader(pdf_path)
+            except ImportError:
+                raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+
+        meta = reader.metadata or {}
+        created = self._parse_pdf_date(meta.get('/CreationDate', ''))
+        modified = self._parse_pdf_date(meta.get('/ModDate', ''))
+        
+        return PDFMetadata(
+            title=meta.get('/Title'),
+            author=meta.get('/Author'),
+            producer=meta.get('/Producer'),
+            created=created,
+            modified=modified,
+            pages=len(reader.pages),
+            encrypted=reader.is_encrypted,
+            file_size=pdf_path.stat().st_size
+        )
+
+    def _parse_pdf_date(self, date_str: str) -> Optional[datetime]:
+        try:
+            match = re.match(r'D:(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})', date_str)
+            if not match:
+                return None
+                
+            return datetime(
+                year=int(match[1]),
+                month=int(match[2]),
+                day=int(match[3]),
+                hour=int(match[4]),
+                minute=int(match[5]),
+                second=int(match[6])
+            )
+        except:
+            return None
+
+# Usage example
+if __name__ == "__main__":
+    import json
+    from pathlib import Path
+    
+    try:
+        # Import PyPDF2 only when running the file directly
+        import PyPDF2
+        from PyPDF2 import PdfReader
+    except ImportError:
+        print("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+        exit(1)
+        
+    current_dir = Path(__file__).resolve().parent
+    pdf_path = f'{current_dir}/test.pdf'
+    
+    strategy = NaivePDFProcessorStrategy()
+    result = strategy.process(Path(pdf_path))
+    
+    # Convert to JSON
+    json_output = asdict(result)
+    print(json.dumps(json_output, indent=2, default=str))
+    
+    with open(f'{current_dir}/test.html', 'w') as f:
+        for page in result.pages:
+            f.write(f'<h1>Page {page["page_number"]}</h1>')
+            f.write(page['html'])
+    with open(f'{current_dir}/test.md', 'w') as f:
+        for page in result.pages:
+            f.write(f'# Page {page["page_number"]}\n\n')
+            f.write(clean_pdf_text(page["page_number"], page['raw_text']))
+            f.write('\n\n')

crawl4ai/processors/pdf/utils.py

@@ -0,0 +1,350 @@
+import re
+
+def apply_png_predictor(data, width, bits, color_channels):
+    """Decode PNG predictor (PDF 1.5+ filter)"""
+    bytes_per_pixel = (bits * color_channels) // 8
+    if (bits * color_channels) % 8 != 0:
+        bytes_per_pixel += 1
+        
+    stride = width * bytes_per_pixel
+    scanline_length = stride + 1  # +1 for filter byte
+    
+    if len(data) % scanline_length != 0:
+        raise ValueError("Invalid scanline structure")
+    
+    num_lines = len(data) // scanline_length
+    output = bytearray()
+    prev_line = b'\x00' * stride
+    
+    for i in range(num_lines):
+        line = data[i*scanline_length:(i+1)*scanline_length]
+        filter_type = line[0]
+        filtered = line[1:]
+        
+        if filter_type == 0:  # None
+            decoded = filtered
+        elif filter_type == 1:  # Sub
+            decoded = bytearray(filtered)
+            for j in range(bytes_per_pixel, len(decoded)):
+                decoded[j] = (decoded[j] + decoded[j - bytes_per_pixel]) % 256
+        elif filter_type == 2:  # Up
+            decoded = bytearray([(filtered[j] + prev_line[j]) % 256 
+                               for j in range(len(filtered))])
+        elif filter_type == 3:  # Average
+            decoded = bytearray(filtered)
+            for j in range(len(decoded)):
+                left = decoded[j - bytes_per_pixel] if j >= bytes_per_pixel else 0
+                up = prev_line[j]
+                avg = (left + up) // 2
+                decoded[j] = (decoded[j] + avg) % 256
+        elif filter_type == 4:  # Paeth
+            decoded = bytearray(filtered)
+            for j in range(len(decoded)):
+                left = decoded[j - bytes_per_pixel] if j >= bytes_per_pixel else 0
+                up = prev_line[j]
+                up_left = prev_line[j - bytes_per_pixel] if j >= bytes_per_pixel else 0
+                paeth = paeth_predictor(left, up, up_left)
+                decoded[j] = (decoded[j] + paeth) % 256
+        else:
+            raise ValueError(f"Unsupported filter type: {filter_type}")
+        
+        output.extend(decoded)
+        prev_line = decoded
+    
+    return bytes(output)
+
+def paeth_predictor(a, b, c):
+    p = a + b - c
+    pa = abs(p - a)
+    pb = abs(p - b)
+    pc = abs(p - c)
+    if pa <= pb and pa <= pc:
+        return a
+    elif pb <= pc:
+        return b
+    else:
+        return c
+
+import re
+import html
+
+def clean_pdf_text_to_html(page_number, text):
+    # Decode Unicode escapes and handle surrogate pairs
+    try:
+        decoded = text.encode('latin-1').decode('unicode-escape')
+        decoded = decoded.encode('utf-16', 'surrogatepass').decode('utf-16')
+    except Exception as e:
+        decoded = text  # Fallback if decoding fails
+    
+    article_title_detected = False
+    # decoded = re.sub(r'\.\n', '.\n\n', decoded)
+    # decoded = re.sub(r'\.\n', '<|break|>', decoded)
+    lines = decoded.split('\n')
+    output = []
+    current_paragraph = []
+    in_header = False
+    email_pattern = re.compile(r'\{.*?\}')
+    affiliation_pattern = re.compile(r'^†')
+    quote_pattern = re.compile(r'^["“]')
+    author_pattern = re.compile(
+        r'^\s*[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?'
+        r'(?:,\s*[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?)*'
+        r'(?:,\s*(?:and|&)\s+[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?)?\s*$'
+    )
+    
+    def flush_paragraph():
+        if current_paragraph:
+            para = ' '.join(current_paragraph)
+            para = re.sub(r'\s+', ' ', para).strip()
+            if para:
+                # escaped_para = html.escape(para)
+                escaped_para = para
+                # escaped_para = re.sub(r'\.\n', '.\n\n', escaped_para)
+                # Split escaped_para by <|break|> to avoid HTML escaping
+                escaped_para = escaped_para.split('.\n\n')
+                # Wrap each part in <p> tag
+                escaped_para = [f'<p>{part}</p>' for part in escaped_para]
+                output.append(f'<div class="paragraph">{"".join(escaped_para)}</div><hr/>')
+            current_paragraph.clear()
+    
+    for i, line in enumerate(lines):
+        line = line.strip()
+        
+        # Handle empty lines
+        if not line:
+            flush_paragraph()
+            continue
+            
+        # Detect article title (first line with reasonable length)
+        if not article_title_detected and i == 0 and 3 <= len(line.split()) <= 8 and len(lines) > 1:
+            flush_paragraph()
+            escaped_line = html.escape(line)
+            output.append(f'<h2>{escaped_line}</h2>')
+            article_title_detected = True
+            continue
+            
+        # Detect numbered headers like "2.1 Background"
+        numbered_header = re.match(r'^(\d+(?:\.\d+)*)\s+(.+)$', line)
+        if i > 0 and not lines[i-1].strip() and numbered_header:
+            flush_paragraph()
+            level = numbered_header.group(1).count('.') + 1
+            header_text = numbered_header.group(2)
+            md_level = min(level + 1, 6)
+            escaped_header = html.escape(header_text)
+            output.append(f'<h{md_level}>{escaped_header}</h{md_level}>')
+            in_header = True
+            continue
+            
+        # Detect authors
+        if page_number == 1 and author_pattern.match(line):
+            authors = re.sub(r'[†â€]', '', line)
+            authors = re.split(r', | and ', authors)
+            formatted_authors = []
+            for author in authors:
+                if author.strip():
+                    parts = [p for p in author.strip().split() if p]
+                    formatted = ' '.join(parts)
+                    escaped_author = html.escape(formatted)
+                    formatted_authors.append(f'<strong>{escaped_author}</strong>')
+            
+            if len(formatted_authors) > 1:
+                joined = ', '.join(formatted_authors[:-1]) + ' and ' + formatted_authors[-1]
+            else:
+                joined = formatted_authors[0]
+            
+            output.append(f'<p>{joined}</p>')
+            continue
+            
+        # Detect affiliation
+        if affiliation_pattern.match(line):
+            escaped_line = html.escape(line)
+            output.append(f'<p><em>{escaped_line}</em></p>')
+            continue
+            
+        # Detect emails
+        if email_pattern.match(line):
+            escaped_line = html.escape(line)
+            output.append(f'<p><code>{escaped_line}</code></p>')
+            continue
+            
+        # Detect section headers
+        if re.match(r'^(Abstract|\d+\s+[A-Z]|References|Appendix|Figure|Table)', line):
+            flush_paragraph()
+            escaped_line = html.escape(line)
+            output.append(f'<h2 class="section-header"><em>{escaped_line}</em></h2>')
+            in_header = True
+            continue
+            
+        # Handle quotes
+        if quote_pattern.match(line):
+            flush_paragraph()
+            escaped_line = html.escape(line)
+            output.append(f'<blockquote><p>{escaped_line}</p></blockquote>')
+            continue
+            
+        # Handle hyphenated words
+        if line.endswith('-'):
+            current_paragraph.append(line[:-1].strip())
+        else:
+            current_paragraph.append(line)
+            
+        # Handle paragraph breaks after headers
+        if in_header and not line.endswith(('.', '!', '?')):
+            flush_paragraph()
+            in_header = False
+    
+    flush_paragraph()
+    
+    # Post-process HTML
+    html_output = '\n'.join(output)
+    
+    # Fix common citation patterns
+    html_output = re.sub(r'\(([A-Z][a-z]+ et al\. \d{4})\)', r'<cite>\1</cite>', html_output)
+    
+    # Fix escaped characters
+    html_output = html_output.replace('\\ud835', '').replace('\\u2020', '†')
+    
+    # Remove leftover hyphens and fix spacing
+    html_output = re.sub(r'\s+-\s+', '', html_output)
+    html_output = re.sub(r'\s+([.,!?)])', r'\1', html_output)
+    
+    return html_output
+
+def clean_pdf_text(page_number, text):
+    # Decode Unicode escapes and handle surrogate pairs
+    try:
+        decoded = text.encode('latin-1').decode('unicode-escape')
+        decoded = decoded.encode('utf-16', 'surrogatepass').decode('utf-16')
+    except Exception as e:
+        decoded = text  # Fallback if decoding fails
+    
+    article_title_detected = False
+    decoded = re.sub(r'\.\n', '.\n\n', decoded)
+    lines = decoded.split('\n')
+    output = []
+    current_paragraph = []
+    in_header = False
+    email_pattern = re.compile(r'\{.*?\}')
+    affiliation_pattern = re.compile(r'^†')
+    quote_pattern = re.compile(r'^["“]')
+    author_pattern = re.compile(
+        r'^\s*[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?'
+        r'(?:,\s*[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?)*'
+        r'(?:,\s*(?:and|&)\s+[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\s*(?:[†*0-9]+)?)?\s*$'
+    )
+    
+    def flush_paragraph():
+        if current_paragraph:
+            para = ' '.join(current_paragraph)
+            para = re.sub(r'\s+', ' ', para).strip()
+            if para:
+                output.append(para)
+            current_paragraph.clear()
+    
+    for i, line in enumerate(lines):
+        line = line.strip()
+        
+        # Handle special patterns
+        if not line:
+            flush_paragraph()
+            continue
+            
+        # Detect headline (first line, reasonable length, surrounded by empty lines)
+        if not article_title_detected and i == 0 and 3 <= len(line.split()) <= 8 and (len(lines) > 1):
+            flush_paragraph()
+            output.append(f'## {line}')
+            continue
+            
+        # Detect paragraph breaks for ALL paragraphs
+        if not line and current_paragraph:
+            flush_paragraph()
+            output.append('')  # Add empty line between paragraphs
+            continue
+                    
+        # Detect numbered headers like "2.1 Background"
+        numbered_header = re.match(r'^(\d+(?:\.\d+)*)\s+(.+)$', line)
+        if not lines[i-1].strip() and numbered_header:
+            flush_paragraph()
+            level = numbered_header.group(1).count('.') + 1  # Convert 2.1 → level 2
+            header_text = numbered_header.group(2)
+            # Never go beyond ### for subsections
+            md_level = min(level + 1, 6)   # 1 → ##, 2 → ###, 3 → #### etc
+            output.append(f'{"#" * md_level} {header_text}')
+            in_header = True
+            continue            
+            
+                    
+        # Detect authors
+        if page_number == 1 and author_pattern.match(line):
+            # Clean and format author names
+            authors = re.sub(r'[†â€]', '', line)  # Remove affiliation markers
+            authors = re.split(r', | and ', authors)
+            formatted_authors = []
+            for author in authors:
+                if author.strip():
+                    # Handle "First Last" formatting
+                    parts = [p for p in author.strip().split() if p]
+                    formatted = ' '.join(parts)
+                    formatted_authors.append(f'**{formatted}**')
+            
+            # Join with commas and "and"
+            if len(formatted_authors) > 1:
+                joined = ', '.join(formatted_authors[:-1]) + ' and ' + formatted_authors[-1]
+            else:
+                joined = formatted_authors[0]
+            
+            output.append(joined)
+            continue
+            
+        # Detect affiliation
+        if affiliation_pattern.match(line):
+            output.append(f'*{line}*')
+            continue
+            
+        # Detect emails
+        if email_pattern.match(line):
+            output.append(f'`{line}`')
+            continue
+            
+        # Detect section headers
+        if re.match(r'^(Abstract|\d+\s+[A-Z]|References|Appendix|Figure|Table)', line):
+            flush_paragraph()
+            output.append(f'_[{line}]_')
+            in_header = True
+            continue
+            
+           
+        # Handle quotes
+        if quote_pattern.match(line):
+            flush_paragraph()
+            output.append(f'> {line}')
+            continue
+            
+        # Handle hyphenated words
+        if line.endswith('-'):
+            current_paragraph.append(line[:-1].strip())
+        else:
+            current_paragraph.append(line)
+            
+        # Handle paragraph breaks after headers
+        if in_header and not line.endswith(('.', '!', '?')):
+            flush_paragraph()
+            in_header = False
+    
+    flush_paragraph()
+    
+    # Post-processing
+    markdown = '\n\n'.join(output)
+    
+    # Fix common citation patterns
+    markdown = re.sub(r'\(([A-Z][a-z]+ et al\. \d{4})\)', r'[\1]', markdown)
+    
+    # Fix escaped characters
+    markdown = markdown.replace('\\ud835', '').replace('\\u2020', '†')
+    
+    # Remove leftover hyphens and fix spacing
+    markdown = re.sub(r'\s+-\s+', '', markdown)  # Join hyphenated words
+    markdown = re.sub(r'\s+([.,!?)])', r'\1', markdown)  # Fix punctuation spacing
+    
+    
+    return markdown

crawl4ai/prompts.py

@@ -198,7 +198,804 @@ Avoid Common Mistakes:
 - Do NOT add any comments using "//" or "#" in the JSON output. It causes parsing errors.
 - Make sure the JSON is properly formatted with curly braces, square brackets, and commas in the right places.
 - Do not miss closing </blocks> tag at the end of the JSON output.
-- Do not generate the Python coee show me how to do the task, this is your task to extract the information and return it in JSON format.
+- Do not generate the Python code show me how to do the task, this is your task to extract the information and return it in JSON format.
 
 Result
 Output the final list of JSON objects, wrapped in <blocks>...</blocks> XML tags. Make sure to close the tag properly."""
+
+
+PROMPT_FILTER_CONTENT = """Your task is to filter and convert HTML content into clean, focused markdown that's optimized for use with LLMs and information retrieval systems.
+
+TASK DETAILS:
+1. Content Selection
+- DO: Keep essential information, main content, key details
+- DO: Preserve hierarchical structure using markdown headers
+- DO: Keep code blocks, tables, key lists
+- DON'T: Include navigation menus, ads, footers, cookie notices
+- DON'T: Keep social media widgets, sidebars, related content
+
+2. Content Transformation
+- DO: Use proper markdown syntax (#, ##, **, `, etc)
+- DO: Convert tables to markdown tables
+- DO: Preserve code formatting with ```language blocks
+- DO: Maintain link texts but remove tracking parameters
+- DON'T: Include HTML tags in output
+- DON'T: Keep class names, ids, or other HTML attributes
+
+3. Content Organization
+- DO: Maintain logical flow of information
+- DO: Group related content under appropriate headers
+- DO: Use consistent header levels
+- DON'T: Fragment related content
+- DON'T: Duplicate information
+
+IMPORTANT: If user specific instruction is provided, ignore above guideline and prioritize those requirements over these general guidelines.
+
+OUTPUT FORMAT: 
+Wrap your response in <content> tags. Use proper markdown throughout.
+<content>
+[Your markdown content here]
+</content>
+
+Begin filtering now.
+
+--------------------------------------------
+
+<|HTML_CONTENT_START|>
+{HTML}
+<|HTML_CONTENT_END|>
+
+<|USER_INSTRUCTION_START|>
+{REQUEST}
+<|USER_INSTRUCTION_END|>
+"""
+
+JSON_SCHEMA_BUILDER= """
+# HTML Schema Generation Instructions
+You are a specialized model designed to analyze HTML patterns and generate extraction schemas. Your primary job is to create structured JSON schemas that can be used to extract data from HTML in a consistent and reliable way. When presented with HTML content, you must analyze its structure and generate a schema that captures all relevant data points.
+
+## Your Core Responsibilities:
+1. Analyze HTML structure to identify repeating patterns and important data points
+2. Generate valid JSON schemas following the specified format
+3. Create appropriate selectors that will work reliably for data extraction
+4. Name fields meaningfully based on their content and purpose
+5. Handle both specific user requests and autonomous pattern detection
+
+## Available Schema Types You Can Generate:
+
+<schema_types>
+1. Basic Single-Level Schema
+   - Use for simple, flat data structures
+   - Example: Product cards, user profiles
+   - Direct field extractions
+
+2. Nested Object Schema
+   - Use for hierarchical data
+   - Example: Articles with author details
+   - Contains objects within objects
+
+3. List Schema
+   - Use for repeating elements
+   - Example: Comment sections, product lists
+   - Handles arrays of similar items
+
+4. Complex Nested Lists
+   - Use for multi-level data
+   - Example: Categories with subcategories
+   - Multiple levels of nesting
+
+5. Transformation Schema
+   - Use for data requiring processing
+   - Supports regex and text transformations
+   - Special attribute handling
+</schema_types>
+
+<schema_structure>
+Your output must always be a JSON object with this structure:
+{
+  "name": "Descriptive name of the pattern",
+  "baseSelector": "CSS selector for the repeating element",
+  "fields": [
+    {
+      "name": "field_name",
+      "selector": "CSS selector",
+      "type": "text|attribute|nested|list|regex",
+      "attribute": "attribute_name",  // Optional
+      "transform": "transformation_type",  // Optional
+      "pattern": "regex_pattern",  // Optional
+      "fields": []  // For nested/list types
+    }
+  ]
+}
+</schema_structure>
+
+<type_definitions>
+Available field types:
+- text: Direct text extraction
+- attribute: HTML attribute extraction
+- nested: Object containing other fields
+- list: Array of similar items
+- regex: Pattern-based extraction
+</type_definitions>
+
+<behavior_rules>
+1. When given a specific query:
+   - Focus on extracting requested data points
+   - Use most specific selectors possible
+   - Include all fields mentioned in the query
+
+2. When no query is provided:
+   - Identify main content areas
+   - Extract all meaningful data points
+   - Use semantic structure to determine importance
+   - Include prices, dates, titles, and other common data types
+
+3. Always:
+   - Use reliable CSS selectors
+   - Handle dynamic class names appropriately
+   - Create descriptive field names
+   - Follow consistent naming conventions
+</behavior_rules>
+
+<examples>
+1. Basic Product Card Example:
+<html>
+<div class="product-card" data-cat-id="electronics" data-subcat-id="laptops">
+  <h2 class="product-title">Gaming Laptop</h2>
+  <span class="price">$999.99</span>
+  <img src="laptop.jpg" alt="Gaming Laptop">
+</div>
+</html>
+
+Generated Schema:
+{
+  "name": "Product Cards",
+  "baseSelector": ".product-card",
+  "baseFields": [
+    {"name": "data_cat_id", "type": "attribute", "attribute": "data-cat-id"},
+    {"name": "data_subcat_id", "type": "attribute", "attribute": "data-subcat-id"}
+  ],
+  "fields": [
+    {
+      "name": "title",
+      "selector": ".product-title",
+      "type": "text"
+    },
+    {
+      "name": "price",
+      "selector": ".price",
+      "type": "text"
+    },
+    {
+      "name": "image_url",
+      "selector": "img",
+      "type": "attribute",
+      "attribute": "src"
+    }
+  ]
+}
+
+2. Article with Author Details Example:
+<html>
+<article>
+  <h1>The Future of AI</h1>
+  <div class="author-info">
+    <span class="author-name">Dr. Smith</span>
+    <img src="author.jpg" alt="Dr. Smith">
+  </div>
+</article>
+</html>
+
+Generated Schema:
+{
+  "name": "Article Details",
+  "baseSelector": "article",
+  "fields": [
+    {
+      "name": "title",
+      "selector": "h1",
+      "type": "text"
+    },
+    {
+      "name": "author",
+      "type": "nested",
+      "selector": ".author-info",
+      "fields": [
+        {
+          "name": "name",
+          "selector": ".author-name",
+          "type": "text"
+        },
+        {
+          "name": "avatar",
+          "selector": "img",
+          "type": "attribute",
+          "attribute": "src"
+        }
+      ]
+    }
+  ]
+}
+
+3. Comments Section Example:
+<html>
+<div class="comments-container">
+  <div class="comment" data-user-id="123">
+    <div class="user-name">John123</div>
+    <p class="comment-text">Great article!</p>
+  </div>
+  <div class="comment" data-user-id="456">
+    <div class="user-name">Alice456</div>
+    <p class="comment-text">Thanks for sharing.</p>
+  </div>
+</div>
+</html>
+
+Generated Schema:
+{
+  "name": "Comment Section",
+  "baseSelector": ".comments-container",
+  "baseFields": [
+    {"name": "data_user_id", "type": "attribute", "attribute": "data-user-id"}
+  ],
+  "fields": [
+    {
+      "name": "comments",
+      "type": "list",
+      "selector": ".comment",
+      "fields": [
+        {
+          "name": "user",
+          "selector": ".user-name",
+          "type": "text"
+        },
+        {
+          "name": "content",
+          "selector": ".comment-text",
+          "type": "text"
+        }
+      ]
+    }
+  ]
+}
+
+4. E-commerce Categories Example:
+<html>
+<div class="category-section" data-category="electronics">
+  <h2>Electronics</h2>
+  <div class="subcategory">
+    <h3>Laptops</h3>
+    <div class="product">
+      <span class="product-name">MacBook Pro</span>
+      <span class="price">$1299</span>
+    </div>
+    <div class="product">
+      <span class="product-name">Dell XPS</span>
+      <span class="price">$999</span>
+    </div>
+  </div>
+</div>
+</html>
+
+Generated Schema:
+{
+  "name": "E-commerce Categories",
+  "baseSelector": ".category-section",
+  "baseFields": [
+    {"name": "data_category", "type": "attribute", "attribute": "data-category"}
+  ],
+  "fields": [
+    {
+      "name": "category_name",
+      "selector": "h2",
+      "type": "text"
+    },
+    {
+      "name": "subcategories",
+      "type": "nested_list",
+      "selector": ".subcategory",
+      "fields": [
+        {
+          "name": "name",
+          "selector": "h3",
+          "type": "text"
+        },
+        {
+          "name": "products",
+          "type": "list",
+          "selector": ".product",
+          "fields": [
+            {
+              "name": "name",
+              "selector": ".product-name",
+              "type": "text"
+            },
+            {
+              "name": "price",
+              "selector": ".price",
+              "type": "text"
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
+
+5. Job Listings with Transformations Example:
+<html>
+<div class="job-post">
+  <h3 class="job-title">Senior Developer</h3>
+  <span class="salary-text">Salary: $120,000/year</span>
+  <span class="location">  New York, NY  </span>
+</div>
+</html>
+
+Generated Schema:
+{
+  "name": "Job Listings",
+  "baseSelector": ".job-post",
+  "fields": [
+    {
+      "name": "title",
+      "selector": ".job-title",
+      "type": "text",
+      "transform": "uppercase"
+    },
+    {
+      "name": "salary",
+      "selector": ".salary-text",
+      "type": "regex",
+      "pattern": "\\$([\\d,]+)"
+    },
+    {
+      "name": "location",
+      "selector": ".location",
+      "type": "text",
+      "transform": "strip"
+    }
+  ]
+}
+
+6. Skyscanner Place Card Example:
+<html>
+<div class="PlaceCard_descriptionContainer__M2NjN" data-testid="description-container">
+  <div class="PlaceCard_nameContainer__ZjZmY" tabindex="0" role="link">
+    <div class="PlaceCard_nameContent__ODUwZ">
+      <span class="BpkText_bpk-text__MjhhY BpkText_bpk-text--heading-4__Y2FlY">Doha</span>
+    </div>
+    <span class="BpkText_bpk-text__MjhhY BpkText_bpk-text--heading-4__Y2FlY PlaceCard_subName__NTVkY">Qatar</span>
+  </div>
+  <span class="PlaceCard_advertLabel__YTM0N">Sunny days and the warmest welcome awaits</span>
+  <a class="BpkLink_bpk-link__MmQwY PlaceCard_descriptionLink__NzYwN" href="/flights/del/doha/" data-testid="flights-link">
+    <div class="PriceDescription_container__NjEzM">
+      <span class="BpkText_bpk-text--heading-5__MTRjZ">₹17,559</span>
+    </div>
+  </a>
+</div>
+</html>
+
+Generated Schema:
+{
+  "name": "Skyscanner Place Cards",
+  "baseSelector": "div[class^='PlaceCard_descriptionContainer__']",
+  "baseFields": [
+    {"name": "data_testid", "type": "attribute", "attribute": "data-testid"}
+  ],
+  "fields": [
+    {
+      "name": "city_name",
+      "selector": "div[class^='PlaceCard_nameContent__'] .BpkText_bpk-text--heading-4__",
+      "type": "text"
+    },
+    {
+      "name": "country_name",
+      "selector": "span[class*='PlaceCard_subName__']",
+      "type": "text"
+    },
+    {
+      "name": "description",
+      "selector": "span[class*='PlaceCard_advertLabel__']",
+      "type": "text"
+    },
+    {
+      "name": "flight_price",
+      "selector": "a[data-testid='flights-link'] .BpkText_bpk-text--heading-5__",
+      "type": "text"
+    },
+    {
+      "name": "flight_url",
+      "selector": "a[data-testid='flights-link']",
+      "type": "attribute",
+      "attribute": "href"
+    }
+  ]
+}
+</examples>
+
+
+<output_requirements>
+Your output must:
+1. Be valid JSON only
+2. Include no explanatory text
+3. Follow the exact schema structure provided
+4. Use appropriate field types
+5. Include all required fields
+6. Use valid CSS selectors
+</output_requirements>
+
+"""
+
+JSON_SCHEMA_BUILDER_XPATH = """
+# HTML Schema Generation Instructions
+You are a specialized model designed to analyze HTML patterns and generate extraction schemas. Your primary job is to create structured JSON schemas that can be used to extract data from HTML in a consistent and reliable way. When presented with HTML content, you must analyze its structure and generate a schema that captures all relevant data points.
+
+## Your Core Responsibilities:
+1. Analyze HTML structure to identify repeating patterns and important data points
+2. Generate valid JSON schemas following the specified format
+3. Create appropriate XPath selectors that will work reliably for data extraction
+4. Name fields meaningfully based on their content and purpose
+5. Handle both specific user requests and autonomous pattern detection
+
+## Available Schema Types You Can Generate:
+
+<schema_types>
+1. Basic Single-Level Schema
+  - Use for simple, flat data structures
+  - Example: Product cards, user profiles
+  - Direct field extractions
+
+2. Nested Object Schema
+  - Use for hierarchical data
+  - Example: Articles with author details
+  - Contains objects within objects
+
+3. List Schema
+  - Use for repeating elements
+  - Example: Comment sections, product lists
+  - Handles arrays of similar items
+
+4. Complex Nested Lists
+  - Use for multi-level data
+  - Example: Categories with subcategories
+  - Multiple levels of nesting
+
+5. Transformation Schema
+  - Use for data requiring processing
+  - Supports regex and text transformations
+  - Special attribute handling
+</schema_types>
+
+<schema_structure>
+Your output must always be a JSON object with this structure:
+{
+ "name": "Descriptive name of the pattern",
+ "baseSelector": "XPath selector for the repeating element",
+ "fields": [
+   {
+     "name": "field_name",
+     "selector": "XPath selector",
+     "type": "text|attribute|nested|list|regex",
+     "attribute": "attribute_name",  // Optional
+     "transform": "transformation_type",  // Optional
+     "pattern": "regex_pattern",  // Optional
+     "fields": []  // For nested/list types
+   }
+ ]
+}
+</schema_structure>
+
+<type_definitions>
+Available field types:
+- text: Direct text extraction
+- attribute: HTML attribute extraction
+- nested: Object containing other fields
+- list: Array of similar items
+- regex: Pattern-based extraction
+</type_definitions>
+
+<behavior_rules>
+1. When given a specific query:
+  - Focus on extracting requested data points
+  - Use most specific selectors possible
+  - Include all fields mentioned in the query
+
+2. When no query is provided:
+  - Identify main content areas
+  - Extract all meaningful data points
+  - Use semantic structure to determine importance
+  - Include prices, dates, titles, and other common data types
+
+3. Always:
+  - Use reliable XPath selectors
+  - Handle dynamic element IDs appropriately
+  - Create descriptive field names
+  - Follow consistent naming conventions
+</behavior_rules>
+
+<examples>
+1. Basic Product Card Example:
+<html>
+<div class="product-card" data-cat-id="electronics" data-subcat-id="laptops">
+ <h2 class="product-title">Gaming Laptop</h2>
+ <span class="price">$999.99</span>
+ <img src="laptop.jpg" alt="Gaming Laptop">
+</div>
+</html>
+
+Generated Schema:
+{
+ "name": "Product Cards",
+ "baseSelector": "//div[@class='product-card']",
+ "baseFields": [
+   {"name": "data_cat_id", "type": "attribute", "attribute": "data-cat-id"},
+   {"name": "data_subcat_id", "type": "attribute", "attribute": "data-subcat-id"}
+ ],
+ "fields": [
+   {
+     "name": "title",
+     "selector": ".//h2[@class='product-title']",
+     "type": "text"
+   },
+   {
+     "name": "price",
+     "selector": ".//span[@class='price']",
+     "type": "text"
+   },
+   {
+     "name": "image_url",
+     "selector": ".//img",
+     "type": "attribute",
+     "attribute": "src"
+   }
+ ]
+}
+
+2. Article with Author Details Example:
+<html>
+<article>
+ <h1>The Future of AI</h1>
+ <div class="author-info">
+   <span class="author-name">Dr. Smith</span>
+   <img src="author.jpg" alt="Dr. Smith">
+ </div>
+</article>
+</html>
+
+Generated Schema:
+{
+ "name": "Article Details",
+ "baseSelector": "//article",
+ "fields": [
+   {
+     "name": "title",
+     "selector": ".//h1",
+     "type": "text"
+   },
+   {
+     "name": "author",
+     "type": "nested",
+     "selector": ".//div[@class='author-info']",
+     "fields": [
+       {
+         "name": "name",
+         "selector": ".//span[@class='author-name']",
+         "type": "text"
+       },
+       {
+         "name": "avatar",
+         "selector": ".//img",
+         "type": "attribute",
+         "attribute": "src"
+       }
+     ]
+   }
+ ]
+}
+
+3. Comments Section Example:
+<html>
+<div class="comments-container">
+ <div class="comment" data-user-id="123">
+   <div class="user-name">John123</div>
+   <p class="comment-text">Great article!</p>
+ </div>
+ <div class="comment" data-user-id="456">
+   <div class="user-name">Alice456</div>
+   <p class="comment-text">Thanks for sharing.</p>
+ </div>
+</div>
+</html>
+
+Generated Schema:
+{
+ "name": "Comment Section",
+ "baseSelector": "//div[@class='comments-container']",
+ "fields": [
+   {
+     "name": "comments",
+     "type": "list",
+     "selector": ".//div[@class='comment']",
+     "baseFields": [
+       {"name": "data_user_id", "type": "attribute", "attribute": "data-user-id"}
+     ],
+     "fields": [
+       {
+         "name": "user",
+         "selector": ".//div[@class='user-name']",
+         "type": "text"
+       },
+       {
+         "name": "content",
+         "selector": ".//p[@class='comment-text']",
+         "type": "text"
+       }
+     ]
+   }
+ ]
+}
+
+4. E-commerce Categories Example:
+<html>
+<div class="category-section" data-category="electronics">
+ <h2>Electronics</h2>
+ <div class="subcategory">
+   <h3>Laptops</h3>
+   <div class="product">
+     <span class="product-name">MacBook Pro</span>
+     <span class="price">$1299</span>
+   </div>
+   <div class="product">
+     <span class="product-name">Dell XPS</span>
+     <span class="price">$999</span>
+   </div>
+ </div>
+</div>
+</html>
+
+Generated Schema:
+{
+ "name": "E-commerce Categories",
+ "baseSelector": "//div[@class='category-section']",
+ "baseFields": [
+   {"name": "data_category", "type": "attribute", "attribute": "data-category"}
+ ],
+ "fields": [
+   {
+     "name": "category_name",
+     "selector": ".//h2",
+     "type": "text"
+   },
+   {
+     "name": "subcategories",
+     "type": "nested_list",
+     "selector": ".//div[@class='subcategory']",
+     "fields": [
+       {
+         "name": "name",
+         "selector": ".//h3",
+         "type": "text"
+       },
+       {
+         "name": "products",
+         "type": "list",
+         "selector": ".//div[@class='product']",
+         "fields": [
+           {
+             "name": "name",
+             "selector": ".//span[@class='product-name']",
+             "type": "text"
+           },
+           {
+             "name": "price",
+             "selector": ".//span[@class='price']",
+             "type": "text"
+           }
+         ]
+       }
+     ]
+   }
+ ]
+}
+
+5. Job Listings with Transformations Example:
+<html>
+<div class="job-post">
+ <h3 class="job-title">Senior Developer</h3>
+ <span class="salary-text">Salary: $120,000/year</span>
+ <span class="location">  New York, NY  </span>
+</div>
+</html>
+
+Generated Schema:
+{
+ "name": "Job Listings",
+ "baseSelector": "//div[@class='job-post']",
+ "fields": [
+   {
+     "name": "title",
+     "selector": ".//h3[@class='job-title']",
+     "type": "text",
+     "transform": "uppercase"
+   },
+   {
+     "name": "salary",
+     "selector": ".//span[@class='salary-text']",
+     "type": "regex",
+     "pattern": "\\$([\\d,]+)"
+   },
+   {
+     "name": "location",
+     "selector": ".//span[@class='location']",
+     "type": "text",
+     "transform": "strip"
+   }
+ ]
+}
+
+6. Skyscanner Place Card Example:
+<html>
+<div class="PlaceCard_descriptionContainer__M2NjN" data-testid="description-container">
+ <div class="PlaceCard_nameContainer__ZjZmY" tabindex="0" role="link">
+   <div class="PlaceCard_nameContent__ODUwZ">
+     <span class="BpkText_bpk-text__MjhhY BpkText_bpk-text--heading-4__Y2FlY">Doha</span>
+   </div>
+   <span class="BpkText_bpk-text__MjhhY BpkText_bpk-text--heading-4__Y2FlY PlaceCard_subName__NTVkY">Qatar</span>
+ </div>
+ <span class="PlaceCard_advertLabel__YTM0N">Sunny days and the warmest welcome awaits</span>
+ <a class="BpkLink_bpk-link__MmQwY PlaceCard_descriptionLink__NzYwN" href="/flights/del/doha/" data-testid="flights-link">
+   <div class="PriceDescription_container__NjEzM">
+     <span class="BpkText_bpk-text--heading-5__MTRjZ">₹17,559</span>
+   </div>
+ </a>
+</div>
+</html>
+
+Generated Schema:
+{
+ "name": "Skyscanner Place Cards",
+ "baseSelector": "//div[contains(@class, 'PlaceCard_descriptionContainer__')]",
+ "baseFields": [
+   {"name": "data_testid", "type": "attribute", "attribute": "data-testid"}
+ ],
+ "fields": [
+   {
+     "name": "city_name",
+     "selector": ".//div[contains(@class, 'PlaceCard_nameContent__')]//span[contains(@class, 'BpkText_bpk-text--heading-4__')]",
+     "type": "text"
+   },
+   {
+     "name": "country_name",
+     "selector": ".//span[contains(@class, 'PlaceCard_subName__')]",
+     "type": "text"
+   },
+   {
+     "name": "description",
+     "selector": ".//span[contains(@class, 'PlaceCard_advertLabel__')]",
+     "type": "text"
+   },
+   {
+     "name": "flight_price",
+     "selector": ".//a[@data-testid='flights-link']//span[contains(@class, 'BpkText_bpk-text--heading-5__')]",
+     "type": "text"
+   },
+   {
+     "name": "flight_url",
+     "selector": ".//a[@data-testid='flights-link']",
+     "type": "attribute",
+     "attribute": "href"
+   }
+ ]
+}
+</examples>
+
+<output_requirements>
+Your output must:
+1. Be valid JSON only
+2. Include no explanatory text
+3. Follow the exact schema structure provided
+4. Use appropriate field types
+5. Include all required fields
+6. Use valid XPath selectors
+</output_requirements>
+"""

crawl4ai/proxy_strategy.py

@@ -0,0 +1,44 @@
+from typing import List, Dict, Optional
+from abc import ABC, abstractmethod
+from itertools import cycle
+
+from crawl4ai.configs import ProxyConfig
+
+class ProxyRotationStrategy(ABC):
+    """Base abstract class for proxy rotation strategies"""
+    
+    @abstractmethod
+    async def get_next_proxy(self) -> Optional[Dict]:
+        """Get next proxy configuration from the strategy"""
+        pass
+
+    @abstractmethod
+    def add_proxies(self, proxies: List[Dict]):
+        """Add proxy configurations to the strategy"""
+        pass
+
+class RoundRobinProxyStrategy:
+    """Simple round-robin proxy rotation strategy using ProxyConfig objects"""
+
+    def __init__(self, proxies: List[ProxyConfig] = None):
+        """
+        Initialize with optional list of proxy configurations
+        
+        Args:
+            proxies: List of ProxyConfig objects
+        """
+        self._proxies = []
+        self._proxy_cycle = None
+        if proxies:
+            self.add_proxies(proxies)
+
+    def add_proxies(self, proxies: List[ProxyConfig]):
+        """Add new proxies to the rotation pool"""
+        self._proxies.extend(proxies)
+        self._proxy_cycle = cycle(self._proxies)
+
+    async def get_next_proxy(self) -> Optional[ProxyConfig]:
+        """Get next proxy in round-robin fashion"""
+        if not self._proxy_cycle:
+            return None
+        return next(self._proxy_cycle)

crawl4ai/ssl_certificate.py

@@ -0,0 +1,184 @@
+"""SSL Certificate class for handling certificate operations."""
+
+import ssl
+import socket
+import base64
+import json
+from typing import Dict, Any, Optional
+from urllib.parse import urlparse
+import OpenSSL.crypto
+from pathlib import Path
+
+
+class SSLCertificate:
+    """
+    A class representing an SSL certificate with methods to export in various formats.
+
+    Attributes:
+        cert_info (Dict[str, Any]): The certificate information.
+
+        Methods:
+            from_url(url: str, timeout: int = 10) -> Optional['SSLCertificate']: Create SSLCertificate instance from a URL.
+            from_file(file_path: str) -> Optional['SSLCertificate']: Create SSLCertificate instance from a file.
+            from_binary(binary_data: bytes) -> Optional['SSLCertificate']: Create SSLCertificate instance from binary data.
+            export_as_pem() -> str: Export the certificate as PEM format.
+            export_as_der() -> bytes: Export the certificate as DER format.
+            export_as_json() -> Dict[str, Any]: Export the certificate as JSON format.
+            export_as_text() -> str: Export the certificate as text format.
+    """
+
+    def __init__(self, cert_info: Dict[str, Any]):
+        self._cert_info = self._decode_cert_data(cert_info)
+
+    @staticmethod
+    def from_url(url: str, timeout: int = 10) -> Optional["SSLCertificate"]:
+        """
+        Create SSLCertificate instance from a URL.
+
+        Args:
+            url (str): URL of the website.
+            timeout (int): Timeout for the connection (default: 10).
+
+        Returns:
+            Optional[SSLCertificate]: SSLCertificate instance if successful, None otherwise.
+        """
+        try:
+            hostname = urlparse(url).netloc
+            if ":" in hostname:
+                hostname = hostname.split(":")[0]
+
+            context = ssl.create_default_context()
+            with socket.create_connection((hostname, 443), timeout=timeout) as sock:
+                with context.wrap_socket(sock, server_hostname=hostname) as ssock:
+                    cert_binary = ssock.getpeercert(binary_form=True)
+                    x509 = OpenSSL.crypto.load_certificate(
+                        OpenSSL.crypto.FILETYPE_ASN1, cert_binary
+                    )
+
+                    cert_info = {
+                        "subject": dict(x509.get_subject().get_components()),
+                        "issuer": dict(x509.get_issuer().get_components()),
+                        "version": x509.get_version(),
+                        "serial_number": hex(x509.get_serial_number()),
+                        "not_before": x509.get_notBefore(),
+                        "not_after": x509.get_notAfter(),
+                        "fingerprint": x509.digest("sha256").hex(),
+                        "signature_algorithm": x509.get_signature_algorithm(),
+                        "raw_cert": base64.b64encode(cert_binary),
+                    }
+
+                    # Add extensions
+                    extensions = []
+                    for i in range(x509.get_extension_count()):
+                        ext = x509.get_extension(i)
+                        extensions.append(
+                            {"name": ext.get_short_name(), "value": str(ext)}
+                        )
+                    cert_info["extensions"] = extensions
+
+                    return SSLCertificate(cert_info)
+
+        except Exception:
+            return None
+
+    @staticmethod
+    def _decode_cert_data(data: Any) -> Any:
+        """Helper method to decode bytes in certificate data."""
+        if isinstance(data, bytes):
+            return data.decode("utf-8")
+        elif isinstance(data, dict):
+            return {
+                (
+                    k.decode("utf-8") if isinstance(k, bytes) else k
+                ): SSLCertificate._decode_cert_data(v)
+                for k, v in data.items()
+            }
+        elif isinstance(data, list):
+            return [SSLCertificate._decode_cert_data(item) for item in data]
+        return data
+
+    def to_json(self, filepath: Optional[str] = None) -> Optional[str]:
+        """
+        Export certificate as JSON.
+
+        Args:
+            filepath (Optional[str]): Path to save the JSON file (default: None).
+
+        Returns:
+            Optional[str]: JSON string if successful, None otherwise.
+        """
+        json_str = json.dumps(self._cert_info, indent=2, ensure_ascii=False)
+        if filepath:
+            Path(filepath).write_text(json_str, encoding="utf-8")
+            return None
+        return json_str
+
+    def to_pem(self, filepath: Optional[str] = None) -> Optional[str]:
+        """
+        Export certificate as PEM.
+
+        Args:
+            filepath (Optional[str]): Path to save the PEM file (default: None).
+
+        Returns:
+            Optional[str]: PEM string if successful, None otherwise.
+        """
+        try:
+            x509 = OpenSSL.crypto.load_certificate(
+                OpenSSL.crypto.FILETYPE_ASN1,
+                base64.b64decode(self._cert_info["raw_cert"]),
+            )
+            pem_data = OpenSSL.crypto.dump_certificate(
+                OpenSSL.crypto.FILETYPE_PEM, x509
+            ).decode("utf-8")
+
+            if filepath:
+                Path(filepath).write_text(pem_data, encoding="utf-8")
+                return None
+            return pem_data
+        except Exception:
+            return None
+
+    def to_der(self, filepath: Optional[str] = None) -> Optional[bytes]:
+        """
+        Export certificate as DER.
+
+        Args:
+            filepath (Optional[str]): Path to save the DER file (default: None).
+
+        Returns:
+            Optional[bytes]: DER bytes if successful, None otherwise.
+        """
+        try:
+            der_data = base64.b64decode(self._cert_info["raw_cert"])
+            if filepath:
+                Path(filepath).write_bytes(der_data)
+                return None
+            return der_data
+        except Exception:
+            return None
+
+    @property
+    def issuer(self) -> Dict[str, str]:
+        """Get certificate issuer information."""
+        return self._cert_info.get("issuer", {})
+
+    @property
+    def subject(self) -> Dict[str, str]:
+        """Get certificate subject information."""
+        return self._cert_info.get("subject", {})
+
+    @property
+    def valid_from(self) -> str:
+        """Get certificate validity start date."""
+        return self._cert_info.get("not_before", "")
+
+    @property
+    def valid_until(self) -> str:
+        """Get certificate validity end date."""
+        return self._cert_info.get("not_after", "")
+
+    @property
+    def fingerprint(self) -> str:
+        """Get certificate fingerprint."""
+        return self._cert_info.get("fingerprint", "")

crawl4ai/tools.py

@@ -1,34 +0,0 @@
-import time
-import cProfile
-import pstats
-from functools import wraps
-
-def profile_and_time(func):
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        # Start timer
-        start_time = time.perf_counter()
-        
-        # Setup profiler
-        profiler = cProfile.Profile()
-        profiler.enable()
-        
-        # Run function
-        result = func(self, *args, **kwargs)
-        
-        # Stop profiler
-        profiler.disable()
-        
-        # Calculate elapsed time
-        elapsed_time = time.perf_counter() - start_time
-        
-        # Print timing
-        print(f"[PROFILER] Scraping completed in {elapsed_time:.2f} seconds")
-        
-        # Print profiling stats
-        stats = pstats.Stats(profiler)
-        stats.sort_stats('cumulative')  # Sort by cumulative time
-        stats.print_stats(20)  # Print top 20 time-consuming functions
-        
-        return result
-    return wrapper

crawl4ai/types.py

@@ -0,0 +1,14 @@
+from typing import TYPE_CHECKING, Union
+
+AsyncWebCrawler = Union['AsyncWebCrawlerType']  # Note the string literal
+CrawlerRunConfig = Union['CrawlerRunConfigType']
+CrawlResult = Union['CrawlResultType']
+RunManyReturn = Union['RunManyReturnType']
+
+if TYPE_CHECKING:
+    from . import (
+        AsyncWebCrawler as AsyncWebCrawlerType,
+        CrawlerRunConfig as CrawlerRunConfigType,
+        CrawlResult as CrawlResultType,
+        RunManyReturn as RunManyReturnType,
+    )

crawl4ai/user_agent_generator.py

@@ -0,0 +1,428 @@
+import random
+from typing import Optional, Literal, List, Dict, Tuple
+import re
+
+from abc import ABC, abstractmethod
+from fake_useragent import UserAgent
+import requests
+from lxml import html
+import json
+from typing import Union
+
+class UAGen(ABC):
+   @abstractmethod
+   def generate(self, 
+               browsers: Optional[List[str]] = None,
+               os: Optional[Union[str, List[str]]] = None,
+               min_version: float = 0.0,
+               platforms: Optional[Union[str, List[str]]] = None,
+               pct_threshold: Optional[float] = None,
+               fallback: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36") -> Union[str, Dict]:
+       pass
+   
+   @staticmethod
+   def generate_client_hints( user_agent: str) -> str:
+        """Generate Sec-CH-UA header value based on user agent string"""
+        def _parse_user_agent(user_agent: str) -> Dict[str, str]:
+            """Parse a user agent string to extract browser and version information"""
+            browsers = {
+                "chrome": r"Chrome/(\d+)",
+                "edge": r"Edg/(\d+)",
+                "safari": r"Version/(\d+)",
+                "firefox": r"Firefox/(\d+)",
+            }
+
+            result = {}
+            for browser, pattern in browsers.items():
+                match = re.search(pattern, user_agent)
+                if match:
+                    result[browser] = match.group(1)
+
+            return result
+        browsers = _parse_user_agent(user_agent)
+
+        # Client hints components
+        hints = []
+
+        # Handle different browser combinations
+        if "chrome" in browsers:
+            hints.append(f'"Chromium";v="{browsers["chrome"]}"')
+            hints.append('"Not_A Brand";v="8"')
+
+            if "edge" in browsers:
+                hints.append(f'"Microsoft Edge";v="{browsers["edge"]}"')
+            else:
+                hints.append(f'"Google Chrome";v="{browsers["chrome"]}"')
+
+        elif "firefox" in browsers:
+            # Firefox doesn't typically send Sec-CH-UA
+            return '""'
+
+        elif "safari" in browsers:
+            # Safari's format for client hints
+            hints.append(f'"Safari";v="{browsers["safari"]}"')
+            hints.append('"Not_A Brand";v="8"')
+
+        return ", ".join(hints)
+
+class ValidUAGenerator(UAGen):
+   def __init__(self):
+       self.ua = UserAgent()
+       
+   def generate(self,
+               browsers: Optional[List[str]] = None,
+               os: Optional[Union[str, List[str]]] = None, 
+               min_version: float = 0.0,
+               platforms: Optional[Union[str, List[str]]] = None,
+               pct_threshold: Optional[float] = None,
+               fallback: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36") -> str:
+       
+       self.ua = UserAgent(
+           browsers=browsers or ['Chrome', 'Firefox', 'Edge'],
+           os=os or ['Windows', 'Mac OS X'],
+           min_version=min_version,
+           platforms=platforms or ['desktop'],
+           fallback=fallback
+       )
+       return self.ua.random
+
+class OnlineUAGenerator(UAGen):
+   def __init__(self):
+       self.agents = []
+       self._fetch_agents()
+       
+   def _fetch_agents(self):
+       try:
+           response = requests.get(
+               'https://www.useragents.me/',
+               timeout=5,
+               headers={'Accept': 'text/html,application/xhtml+xml'}
+           )
+           response.raise_for_status()
+           
+           tree = html.fromstring(response.content)
+           json_text = tree.cssselect('#most-common-desktop-useragents-json-csv > div:nth-child(1) > textarea')[0].text
+           self.agents = json.loads(json_text)
+       except Exception as e:
+           print(f"Error fetching agents: {e}")
+           
+   def generate(self,
+               browsers: Optional[List[str]] = None,
+               os: Optional[Union[str, List[str]]] = None,
+               min_version: float = 0.0,
+               platforms: Optional[Union[str, List[str]]] = None, 
+               pct_threshold: Optional[float] = None,
+               fallback: str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36") -> Dict:
+       
+       if not self.agents:
+           self._fetch_agents()
+           
+       filtered_agents = self.agents
+       
+       if pct_threshold:
+           filtered_agents = [a for a in filtered_agents if a['pct'] >= pct_threshold]
+           
+       if browsers:
+           filtered_agents = [a for a in filtered_agents 
+                            if any(b.lower() in a['ua'].lower() for b in browsers)]
+           
+       if os:
+           os_list = [os] if isinstance(os, str) else os
+           filtered_agents = [a for a in filtered_agents 
+                            if any(o.lower() in a['ua'].lower() for o in os_list)]
+           
+       if platforms:
+           platform_list = [platforms] if isinstance(platforms, str) else platforms
+           filtered_agents = [a for a in filtered_agents 
+                            if any(p.lower() in a['ua'].lower() for p in platform_list)]
+           
+       return filtered_agents[0] if filtered_agents else {'ua': fallback, 'pct': 0}
+
+
+
+class UserAgentGenerator():
+    """
+    Generate random user agents with specified constraints.
+
+    Attributes:
+        desktop_platforms (dict): A dictionary of possible desktop platforms and their corresponding user agent strings.
+        mobile_platforms (dict): A dictionary of possible mobile platforms and their corresponding user agent strings.
+        browser_combinations (dict): A dictionary of possible browser combinations and their corresponding user agent strings.
+        rendering_engines (dict): A dictionary of possible rendering engines and their corresponding user agent strings.
+        chrome_versions (list): A list of possible Chrome browser versions.
+        firefox_versions (list): A list of possible Firefox browser versions.
+        edge_versions (list): A list of possible Edge browser versions.
+        safari_versions (list): A list of possible Safari browser versions.
+        ios_versions (list): A list of possible iOS browser versions.
+        android_versions (list): A list of possible Android browser versions.
+
+        Methods:
+            generate_user_agent(
+                platform: Literal["desktop", "mobile"] = "desktop",
+                browser: str = "chrome",
+                rendering_engine: str = "chrome_webkit",
+                chrome_version: Optional[str] = None,
+                firefox_version: Optional[str] = None,
+                edge_version: Optional[str] = None,
+                safari_version: Optional[str] = None,
+                ios_version: Optional[str] = None,
+                android_version: Optional[str] = None
+            ): Generates a random user agent string based on the specified parameters.
+    """
+
+    def __init__(self):
+        # Previous platform definitions remain the same...
+        self.desktop_platforms = {
+            "windows": {
+                "10_64": "(Windows NT 10.0; Win64; x64)",
+                "10_32": "(Windows NT 10.0; WOW64)",
+            },
+            "macos": {
+                "intel": "(Macintosh; Intel Mac OS X 10_15_7)",
+                "newer": "(Macintosh; Intel Mac OS X 10.15; rv:109.0)",
+            },
+            "linux": {
+                "generic": "(X11; Linux x86_64)",
+                "ubuntu": "(X11; Ubuntu; Linux x86_64)",
+                "chrome_os": "(X11; CrOS x86_64 14541.0.0)",
+            },
+        }
+
+        self.mobile_platforms = {
+            "android": {
+                "samsung": "(Linux; Android 13; SM-S901B)",
+                "pixel": "(Linux; Android 12; Pixel 6)",
+                "oneplus": "(Linux; Android 13; OnePlus 9 Pro)",
+                "xiaomi": "(Linux; Android 12; M2102J20SG)",
+            },
+            "ios": {
+                "iphone": "(iPhone; CPU iPhone OS 16_5 like Mac OS X)",
+                "ipad": "(iPad; CPU OS 16_5 like Mac OS X)",
+            },
+        }
+
+        # Browser Combinations
+        self.browser_combinations = {
+            1: [["chrome"], ["firefox"], ["safari"], ["edge"]],
+            2: [["gecko", "firefox"], ["chrome", "safari"], ["webkit", "safari"]],
+            3: [["chrome", "safari", "edge"], ["webkit", "chrome", "safari"]],
+        }
+
+        # Rendering Engines with versions
+        self.rendering_engines = {
+            "chrome_webkit": "AppleWebKit/537.36",
+            "safari_webkit": "AppleWebKit/605.1.15",
+            "gecko": [  # Added Gecko versions
+                "Gecko/20100101",
+                "Gecko/20100101",  # Firefox usually uses this constant version
+                "Gecko/2010010",
+            ],
+        }
+
+        # Browser Versions
+        self.chrome_versions = [
+            "Chrome/119.0.6045.199",
+            "Chrome/118.0.5993.117",
+            "Chrome/117.0.5938.149",
+            "Chrome/116.0.5845.187",
+            "Chrome/115.0.5790.171",
+        ]
+
+        self.edge_versions = [
+            "Edg/119.0.2151.97",
+            "Edg/118.0.2088.76",
+            "Edg/117.0.2045.47",
+            "Edg/116.0.1938.81",
+            "Edg/115.0.1901.203",
+        ]
+
+        self.safari_versions = [
+            "Safari/537.36",  # For Chrome-based
+            "Safari/605.1.15",
+            "Safari/604.1",
+            "Safari/602.1",
+            "Safari/601.5.17",
+        ]
+
+        # Added Firefox versions
+        self.firefox_versions = [
+            "Firefox/119.0",
+            "Firefox/118.0.2",
+            "Firefox/117.0.1",
+            "Firefox/116.0",
+            "Firefox/115.0.3",
+            "Firefox/114.0.2",
+            "Firefox/113.0.1",
+            "Firefox/112.0",
+            "Firefox/111.0.1",
+            "Firefox/110.0",
+        ]
+
+    def get_browser_stack(self, num_browsers: int = 1) -> List[str]:
+        """
+        Get a valid combination of browser versions.
+
+        How it works:
+        1. Check if the number of browsers is supported.
+        2. Randomly choose a combination of browsers.
+        3. Iterate through the combination and add browser versions.
+        4. Return the browser stack.
+
+        Args:
+            num_browsers: Number of browser specifications (1-3)
+
+        Returns:
+            List[str]: A list of browser versions.
+        """
+        if num_browsers not in self.browser_combinations:
+            raise ValueError(f"Unsupported number of browsers: {num_browsers}")
+
+        combination = random.choice(self.browser_combinations[num_browsers])
+        browser_stack = []
+
+        for browser in combination:
+            if browser == "chrome":
+                browser_stack.append(random.choice(self.chrome_versions))
+            elif browser == "firefox":
+                browser_stack.append(random.choice(self.firefox_versions))
+            elif browser == "safari":
+                browser_stack.append(random.choice(self.safari_versions))
+            elif browser == "edge":
+                browser_stack.append(random.choice(self.edge_versions))
+            elif browser == "gecko":
+                browser_stack.append(random.choice(self.rendering_engines["gecko"]))
+            elif browser == "webkit":
+                browser_stack.append(self.rendering_engines["chrome_webkit"])
+
+        return browser_stack
+
+    def generate(
+        self,
+        device_type: Optional[Literal["desktop", "mobile"]] = None,
+        os_type: Optional[str] = None,
+        device_brand: Optional[str] = None,
+        browser_type: Optional[Literal["chrome", "edge", "safari", "firefox"]] = None,
+        num_browsers: int = 3,
+    ) -> str:
+        """
+        Generate a random user agent with specified constraints.
+
+        Args:
+            device_type: 'desktop' or 'mobile'
+            os_type: 'windows', 'macos', 'linux', 'android', 'ios'
+            device_brand: Specific device brand
+            browser_type: 'chrome', 'edge', 'safari', or 'firefox'
+            num_browsers: Number of browser specifications (1-3)
+        """
+        # Get platform string
+        platform = self.get_random_platform(device_type, os_type, device_brand)
+
+        # Start with Mozilla
+        components = ["Mozilla/5.0", platform]
+
+        # Add browser stack
+        browser_stack = self.get_browser_stack(num_browsers)
+
+        # Add appropriate legacy token based on browser stack
+        if "Firefox" in str(browser_stack) or browser_type == "firefox":
+            components.append(random.choice(self.rendering_engines["gecko"]))
+        elif "Chrome" in str(browser_stack) or "Safari" in str(browser_stack) or browser_type == "chrome":
+            components.append(self.rendering_engines["chrome_webkit"])
+            components.append("(KHTML, like Gecko)")
+        elif "Edge" in str(browser_stack) or browser_type == "edge":
+            components.append(self.rendering_engines["safari_webkit"])
+            components.append("(KHTML, like Gecko)")
+        elif "Safari" in str(browser_stack) or browser_type == "safari":
+            components.append(self.rendering_engines["chrome_webkit"])
+            components.append("(KHTML, like Gecko)")
+
+        # Add browser versions
+        components.extend(browser_stack)
+
+        return " ".join(components)
+
+    def generate_with_client_hints(self, **kwargs) -> Tuple[str, str]:
+        """Generate both user agent and matching client hints"""
+        user_agent = self.generate(**kwargs)
+        client_hints = self.generate_client_hints(user_agent)
+        return user_agent, client_hints
+
+    def get_random_platform(self, device_type, os_type, device_brand):
+        """Helper method to get random platform based on constraints"""
+        platforms = (
+            self.desktop_platforms
+            if device_type == "desktop"
+            else self.mobile_platforms
+            if device_type == "mobile"
+            else {**self.desktop_platforms, **self.mobile_platforms}
+        )
+
+        if os_type:
+            for platform_group in [self.desktop_platforms, self.mobile_platforms]:
+                if os_type in platform_group:
+                    platforms = {os_type: platform_group[os_type]}
+                    break
+
+        os_key = random.choice(list(platforms.keys()))
+        if device_brand and device_brand in platforms[os_key]:
+            return platforms[os_key][device_brand]
+        return random.choice(list(platforms[os_key].values()))
+
+    def parse_user_agent(self, user_agent: str) -> Dict[str, str]:
+        """Parse a user agent string to extract browser and version information"""
+        browsers = {
+            "chrome": r"Chrome/(\d+)",
+            "edge": r"Edg/(\d+)",
+            "safari": r"Version/(\d+)",
+            "firefox": r"Firefox/(\d+)",
+        }
+
+        result = {}
+        for browser, pattern in browsers.items():
+            match = re.search(pattern, user_agent)
+            if match:
+                result[browser] = match.group(1)
+
+        return result
+
+    def generate_client_hints(self, user_agent: str) -> str:
+        """Generate Sec-CH-UA header value based on user agent string"""
+        browsers = self.parse_user_agent(user_agent)
+
+        # Client hints components
+        hints = []
+
+        # Handle different browser combinations
+        if "chrome" in browsers:
+            hints.append(f'"Chromium";v="{browsers["chrome"]}"')
+            hints.append('"Not_A Brand";v="8"')
+
+            if "edge" in browsers:
+                hints.append(f'"Microsoft Edge";v="{browsers["edge"]}"')
+            else:
+                hints.append(f'"Google Chrome";v="{browsers["chrome"]}"')
+
+        elif "firefox" in browsers:
+            # Firefox doesn't typically send Sec-CH-UA
+            return '""'
+
+        elif "safari" in browsers:
+            # Safari's format for client hints
+            hints.append(f'"Safari";v="{browsers["safari"]}"')
+            hints.append('"Not_A Brand";v="8"')
+
+        return ", ".join(hints)
+
+
+# Example usage:
+if __name__ == "__main__":
+    
+    # Usage example:
+    generator = ValidUAGenerator()
+    ua = generator.generate()
+    print(ua)
+    
+    generator = OnlineUAGenerator()
+    ua = generator.generate()
+    print(ua)
+

crawl4ai/web_crawler.py

@@ -1,253 +0,0 @@
-import os, time
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-from pathlib import Path
-
-from .models import UrlModel, CrawlResult
-from .database import init_db, get_cached_url, cache_url, DB_PATH, flush_db
-from .utils import *
-from .chunking_strategy import *
-from .extraction_strategy import *
-from .crawler_strategy import *
-from typing import List
-from concurrent.futures import ThreadPoolExecutor
-from .content_scraping_strategy import WebScrapingStrategy
-from .config import *
-import warnings
-import json
-warnings.filterwarnings("ignore", message='Field "model_name" has conflict with protected namespace "model_".')
-
-
-class WebCrawler:
-    def __init__(self, crawler_strategy: CrawlerStrategy = None, always_by_pass_cache: bool = False, verbose: bool = False):
-        self.crawler_strategy = crawler_strategy or LocalSeleniumCrawlerStrategy(verbose=verbose)
-        self.always_by_pass_cache = always_by_pass_cache
-        self.crawl4ai_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
-        os.makedirs(self.crawl4ai_folder, exist_ok=True)
-        os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True)
-        init_db()
-        self.ready = False
-        
-    def warmup(self):
-        print("[LOG] 🌤️  Warming up the WebCrawler")
-        self.run(
-            url='https://google.com/',
-            word_count_threshold=5,
-            extraction_strategy=NoExtractionStrategy(),
-            bypass_cache=False,
-            verbose=False
-        )
-        self.ready = True
-        print("[LOG] 🌞 WebCrawler is ready to crawl")
-        
-    def fetch_page(
-        self,
-        url_model: UrlModel,
-        provider: str = DEFAULT_PROVIDER,
-        api_token: str = None,
-        extract_blocks_flag: bool = True,
-        word_count_threshold=MIN_WORD_THRESHOLD,
-        css_selector: str = None,
-        screenshot: bool = False,
-        use_cached_html: bool = False,
-        extraction_strategy: ExtractionStrategy = None,
-        chunking_strategy: ChunkingStrategy = RegexChunking(),
-        **kwargs,
-    ) -> CrawlResult:
-        return self.run(
-            url_model.url,
-            word_count_threshold,
-            extraction_strategy or NoExtractionStrategy(),
-            chunking_strategy,
-            bypass_cache=url_model.forced,
-            css_selector=css_selector,
-            screenshot=screenshot,
-            **kwargs,
-        )
-        pass
-
-    def fetch_pages(
-        self,
-        url_models: List[UrlModel],
-        provider: str = DEFAULT_PROVIDER,
-        api_token: str = None,
-        extract_blocks_flag: bool = True,
-        word_count_threshold=MIN_WORD_THRESHOLD,
-        use_cached_html: bool = False,
-        css_selector: str = None,
-        screenshot: bool = False,
-        extraction_strategy: ExtractionStrategy = None,
-        chunking_strategy: ChunkingStrategy = RegexChunking(),
-        **kwargs,
-    ) -> List[CrawlResult]:
-        extraction_strategy = extraction_strategy or NoExtractionStrategy()
-        def fetch_page_wrapper(url_model, *args, **kwargs):
-            return self.fetch_page(url_model, *args, **kwargs)
-
-        with ThreadPoolExecutor() as executor:
-            results = list(
-                executor.map(
-                    fetch_page_wrapper,
-                    url_models,
-                    [provider] * len(url_models),
-                    [api_token] * len(url_models),
-                    [extract_blocks_flag] * len(url_models),
-                    [word_count_threshold] * len(url_models),
-                    [css_selector] * len(url_models),
-                    [screenshot] * len(url_models),
-                    [use_cached_html] * len(url_models),
-                    [extraction_strategy] * len(url_models),
-                    [chunking_strategy] * len(url_models),
-                    *[kwargs] * len(url_models),
-                )
-            )
-
-        return results
-
-    def run(
-            self,
-            url: str,
-            word_count_threshold=MIN_WORD_THRESHOLD,
-            extraction_strategy: ExtractionStrategy = None,
-            chunking_strategy: ChunkingStrategy = RegexChunking(),
-            bypass_cache: bool = False,
-            css_selector: str = None,
-            screenshot: bool = False,
-            user_agent: str = None,
-            verbose=True,
-            **kwargs,
-        ) -> CrawlResult:
-            try:
-                extraction_strategy = extraction_strategy or NoExtractionStrategy()
-                extraction_strategy.verbose = verbose
-                if not isinstance(extraction_strategy, ExtractionStrategy):
-                    raise ValueError("Unsupported extraction strategy")
-                if not isinstance(chunking_strategy, ChunkingStrategy):
-                    raise ValueError("Unsupported chunking strategy")
-                
-                word_count_threshold = max(word_count_threshold, MIN_WORD_THRESHOLD)
-
-                cached = None
-                screenshot_data = None
-                extracted_content = None
-                if not bypass_cache and not self.always_by_pass_cache:
-                    cached = get_cached_url(url)
-                
-                if kwargs.get("warmup", True) and not self.ready:
-                    return None
-                
-                if cached:
-                    html = sanitize_input_encode(cached[1])
-                    extracted_content = sanitize_input_encode(cached[4])
-                    if screenshot:
-                        screenshot_data = cached[9]
-                        if not screenshot_data:
-                            cached = None
-                
-                if not cached or not html:
-                    if user_agent:
-                        self.crawler_strategy.update_user_agent(user_agent)
-                    t1 = time.time()
-                    html = sanitize_input_encode(self.crawler_strategy.crawl(url, **kwargs))
-                    t2 = time.time()
-                    if verbose:
-                        print(f"[LOG] 🚀 Crawling done for {url}, success: {bool(html)}, time taken: {t2 - t1:.2f} seconds")
-                    if screenshot:
-                        screenshot_data = self.crawler_strategy.take_screenshot()
-
-                
-                crawl_result = self.process_html(url, html, extracted_content, word_count_threshold, extraction_strategy, chunking_strategy, css_selector, screenshot_data, verbose, bool(cached), **kwargs)
-                crawl_result.success = bool(html)
-                return crawl_result
-            except Exception as e:
-                if not hasattr(e, "msg"):
-                    e.msg = str(e)
-                print(f"[ERROR] 🚫 Failed to crawl {url}, error: {e.msg}")    
-                return CrawlResult(url=url, html="", success=False, error_message=e.msg)
-
-    def process_html(
-            self,
-            url: str,
-            html: str,
-            extracted_content: str,
-            word_count_threshold: int,
-            extraction_strategy: ExtractionStrategy,
-            chunking_strategy: ChunkingStrategy,
-            css_selector: str,
-            screenshot: bool,
-            verbose: bool,
-            is_cached: bool,
-            **kwargs,
-        ) -> CrawlResult:
-            t = time.time()
-            # Extract content from HTML
-            try:
-                t1 = time.time()
-                scrapping_strategy = WebScrapingStrategy()
-                extra_params = {k: v for k, v in kwargs.items() if k not in ["only_text", "image_description_min_word_threshold"]}
-                result = scrapping_strategy.scrap(
-                    url,
-                    html,
-                    word_count_threshold=word_count_threshold,
-                    css_selector=css_selector,
-                    only_text=kwargs.get("only_text", False),
-                    image_description_min_word_threshold=kwargs.get(
-                        "image_description_min_word_threshold", IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD
-                    ),
-                    **extra_params,
-                )
-                
-                # result = get_content_of_website_optimized(url, html, word_count_threshold, css_selector=css_selector, only_text=kwargs.get("only_text", False))
-                if verbose:
-                    print(f"[LOG] 🚀 Content extracted for {url}, success: True, time taken: {time.time() - t1:.2f} seconds")
-                
-                if result is None:
-                    raise ValueError(f"Failed to extract content from the website: {url}")
-            except InvalidCSSSelectorError as e:
-                raise ValueError(str(e))
-            
-            cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
-            markdown = sanitize_input_encode(result.get("markdown", ""))
-            media = result.get("media", [])
-            links = result.get("links", [])
-            metadata = result.get("metadata", {})
-                        
-            if extracted_content is None:
-                if verbose:
-                    print(f"[LOG] 🔥 Extracting semantic blocks for {url}, Strategy: {extraction_strategy.name}")
-
-                sections = chunking_strategy.chunk(markdown)
-                extracted_content = extraction_strategy.run(url, sections)
-                extracted_content = json.dumps(extracted_content, indent=4, default=str, ensure_ascii=False)
-
-                if verbose:
-                    print(f"[LOG] 🚀 Extraction done for {url}, time taken: {time.time() - t:.2f} seconds.")
-                
-            screenshot = None if not screenshot else screenshot
-            
-            if not is_cached:
-                cache_url(
-                    url,
-                    html,
-                    cleaned_html,
-                    markdown,
-                    extracted_content,
-                    True,
-                    json.dumps(media),
-                    json.dumps(links),
-                    json.dumps(metadata),
-                    screenshot=screenshot,
-                )                
-            
-            return CrawlResult(
-                url=url,
-                html=html,
-                cleaned_html=format_html(cleaned_html),
-                markdown=markdown,
-                media=media,
-                links=links,
-                metadata=metadata,
-                screenshot=screenshot,
-                extracted_content=extracted_content,
-                success=True,
-                error_message="",
-            )

deploy/docker/.dockerignore

@@ -0,0 +1,31 @@
+# .dockerignore
+*
+
+# Allow specific files and directories when using local installation
+!crawl4ai/
+!docs/
+!deploy/docker/
+!setup.py
+!pyproject.toml
+!README.md
+!LICENSE
+!MANIFEST.in
+!setup.cfg
+!mkdocs.yml
+
+.git/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.DS_Store
+.env
+.venv
+venv/
+tests/
+coverage.xml
+*.log
+*.swp
+*.egg-info/
+dist/
+build/

deploy/docker/.llm.env.example

@@ -0,0 +1,8 @@
+# LLM Provider Keys
+OPENAI_API_KEY=your_openai_key_here
+DEEPSEEK_API_KEY=your_deepseek_key_here
+ANTHROPIC_API_KEY=your_anthropic_key_here
+GROQ_API_KEY=your_groq_key_here
+TOGETHER_API_KEY=your_together_key_here
+MISTRAL_API_KEY=your_mistral_key_here
+GEMINI_API_TOKEN=your_gemini_key_here

deploy/docker/README.md

@@ -0,0 +1,830 @@
+# Crawl4AI Docker Guide 🐳
+
+## Table of Contents
+- [Prerequisites](#prerequisites)
+- [Installation](#installation)
+  - [Local Build](#local-build)
+  - [Docker Hub](#docker-hub)
+- [Dockerfile Parameters](#dockerfile-parameters)
+- [Using the API](#using-the-api)
+  - [Understanding Request Schema](#understanding-request-schema)
+  - [REST API Examples](#rest-api-examples)
+  - [Python SDK](#python-sdk)
+- [Metrics & Monitoring](#metrics--monitoring)
+- [Deployment Scenarios](#deployment-scenarios)
+- [Complete Examples](#complete-examples)
+- [Getting Help](#getting-help)
+
+## Prerequisites
+
+Before we dive in, make sure you have:
+- Docker installed and running (version 20.10.0 or higher)
+- At least 4GB of RAM available for the container
+- Python 3.10+ (if using the Python SDK)
+- Node.js 16+ (if using the Node.js examples)
+
+> 💡 **Pro tip**: Run `docker info` to check your Docker installation and available resources.
+
+## Installation
+
+### Local Build
+
+Let's get your local environment set up step by step!
+
+#### 1. Building the Image
+
+First, clone the repository and build the Docker image:
+
+```bash
+# Clone the repository
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai/deploy
+
+# Build the Docker image
+docker build --platform=linux/amd64 --no-cache -t crawl4ai .
+
+# Or build for arm64
+docker build --platform=linux/arm64 --no-cache -t crawl4ai .
+```
+
+#### 2. Environment Setup
+
+If you plan to use LLMs (Language Models), you'll need to set up your API keys. Create a `.llm.env` file:
+
+```env
+# OpenAI
+OPENAI_API_KEY=sk-your-key
+
+# Anthropic
+ANTHROPIC_API_KEY=your-anthropic-key
+
+# DeepSeek
+DEEPSEEK_API_KEY=your-deepseek-key
+
+# Check out https://docs.litellm.ai/docs/providers for more providers!
+```
+
+> 🔑 **Note**: Keep your API keys secure! Never commit them to version control.
+
+#### 3. Running the Container
+
+You have several options for running the container:
+
+Basic run (no LLM support):
+```bash
+docker run -d -p 8000:8000 --name crawl4ai crawl4ai
+```
+
+With LLM support:
+```bash
+docker run -d -p 8000:8000 \
+  --env-file .llm.env \
+  --name crawl4ai \
+  crawl4ai
+```
+
+Using host environment variables (Not a good practice, but works for local testing):
+```bash
+docker run -d -p 8000:8000 \
+  --env-file .llm.env \
+  --env "$(env)" \
+  --name crawl4ai \
+  crawl4ai
+```
+
+#### Multi-Platform Build
+For distributing your image across different architectures, use `buildx`:
+
+```bash
+# Set up buildx builder
+docker buildx create --use
+
+# Build for multiple platforms
+docker buildx build \
+  --platform linux/amd64,linux/arm64 \
+  -t crawl4ai \
+  --push \
+  .
+```
+
+> 💡 **Note**: Multi-platform builds require Docker Buildx and need to be pushed to a registry.
+
+#### Development Build
+For development, you might want to enable all features:
+
+```bash
+docker build -t crawl4ai
+  --build-arg INSTALL_TYPE=all \
+  --build-arg PYTHON_VERSION=3.10 \
+  --build-arg ENABLE_GPU=true \
+  .
+```
+
+#### GPU-Enabled Build
+If you plan to use GPU acceleration:
+
+```bash
+docker build -t crawl4ai
+  --build-arg ENABLE_GPU=true \
+  deploy/docker/
+```
+
+### Build Arguments Explained
+
+| Argument | Description | Default | Options |
+|----------|-------------|---------|----------|
+| PYTHON_VERSION | Python version | 3.10 | 3.8, 3.9, 3.10 |
+| INSTALL_TYPE | Feature set | default | default, all, torch, transformer |
+| ENABLE_GPU | GPU support | false | true, false |
+| APP_HOME | Install path | /app | any valid path |
+
+### Build Best Practices
+
+1. **Choose the Right Install Type**
+   - `default`: Basic installation, smallest image, to be honest, I use this most of the time.
+   - `all`: Full features, larger image (include transformer, and nltk, make sure you really need them)
+
+2. **Platform Considerations**
+   - Let Docker auto-detect platform unless you need cross-compilation
+   - Use --platform for specific architecture requirements
+   - Consider buildx for multi-architecture distribution
+
+3. **Performance Optimization**
+   - The image automatically includes platform-specific optimizations
+   - AMD64 gets OpenMP optimizations
+   - ARM64 gets OpenBLAS optimizations
+
+### Docker Hub
+
+> 🚧 Coming soon! The image will be available at `crawl4ai`. Stay tuned!
+
+## Using the API
+
+In the following sections, we discuss two ways to communicate with the Docker server. One option is to use the client SDK that I developed for Python, and I will soon develop one for Node.js. I highly recommend this approach to avoid mistakes. Alternatively, you can take a more technical route by using the JSON structure and passing it to all the URLs, which I will explain in detail.
+
+### Python SDK
+
+The SDK makes things easier! Here's how to use it:
+
+```python
+from crawl4ai.docker_client import Crawl4aiDockerClient
+from crawl4ai import BrowserConfig, CrawlerRunConfig
+
+async def main():
+    async with Crawl4aiDockerClient(base_url="http://localhost:8000", verbose=True) as client:
+      # If JWT is enabled, you can authenticate like this: (more on this later)
+        # await client.authenticate("test@example.com")
+        
+        # Non-streaming crawl
+        results = await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig()
+        )
+        print(f"Non-streaming results: {results}")
+        
+        # Streaming crawl
+        crawler_config = CrawlerRunConfig(stream=True)
+        async for result in await client.crawl(
+            ["https://example.com", "https://python.org"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=crawler_config
+        ):
+            print(f"Streamed result: {result}")
+        
+        # Get schema
+        schema = await client.get_schema()
+        print(f"Schema: {schema}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+`Crawl4aiDockerClient` is an async context manager that handles the connection for you. You can pass in optional parameters for more control:
+
+- `base_url` (str): Base URL of the Crawl4AI Docker server
+- `timeout` (float): Default timeout for requests in seconds
+- `verify_ssl` (bool): Whether to verify SSL certificates
+- `verbose` (bool): Whether to show logging output
+- `log_file` (str, optional): Path to log file if file logging is desired
+
+This client SDK generates a properly structured JSON request for the server's HTTP API.
+
+## Second Approach: Direct API Calls
+
+This is super important! The API expects a specific structure that matches our Python classes. Let me show you how it works.
+
+### Understanding Configuration Structure
+
+Let's dive deep into how configurations work in Crawl4AI. Every configuration object follows a consistent pattern of `type` and `params`. This structure enables complex, nested configurations while maintaining clarity.
+
+#### The Basic Pattern
+
+Try this in Python to understand the structure:
+```python
+from crawl4ai import BrowserConfig
+
+# Create a config and see its structure
+config = BrowserConfig(headless=True)
+print(config.dump())
+```
+
+This outputs:
+```json
+{
+    "type": "BrowserConfig",
+    "params": {
+        "headless": true
+    }
+}
+```
+
+#### Simple vs Complex Values
+
+The structure follows these rules:
+- Simple values (strings, numbers, booleans, lists) are passed directly
+- Complex values (classes, dictionaries) use the type-params pattern
+
+For example, with dictionaries:
+```json
+{
+    "browser_config": {
+        "type": "BrowserConfig",
+        "params": {
+            "headless": true,           // Simple boolean - direct value
+            "viewport": {               // Complex dictionary - needs type-params
+                "type": "dict",
+                "value": {
+                    "width": 1200,
+                    "height": 800
+                }
+            }
+        }
+    }
+}
+```
+
+#### Strategy Pattern and Nesting
+
+Strategies (like chunking or content filtering) demonstrate why we need this structure. Consider this chunking configuration:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "chunking_strategy": {
+                "type": "RegexChunking",      // Strategy implementation
+                "params": {
+                    "patterns": ["\n\n", "\\.\\s+"]
+                }
+            }
+        }
+    }
+}
+```
+
+Here, `chunking_strategy` accepts any chunking implementation. The `type` field tells the system which strategy to use, and `params` configures that specific strategy.
+
+#### Complex Nested Example
+
+Let's look at a more complex example with content filtering:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+This shows how deeply configurations can nest while maintaining a consistent structure.
+
+#### Quick Grammar Overview
+```
+config := {
+    "type": string,
+    "params": {
+        key: simple_value | complex_value
+    }
+}
+
+simple_value := string | number | boolean | [simple_value]
+complex_value := config | dict_value
+
+dict_value := {
+    "type": "dict",
+    "value": object
+}
+```
+
+#### Important Rules 🚨
+
+- Always use the type-params pattern for class instances
+- Use direct values for primitives (numbers, strings, booleans)
+- Wrap dictionaries with {"type": "dict", "value": {...}}
+- Arrays/lists are passed directly without type-params
+- All parameters are optional unless specifically required
+
+#### Pro Tip 💡
+
+The easiest way to get the correct structure is to:
+1. Create configuration objects in Python
+2. Use the `dump()` method to see their JSON representation
+3. Use that JSON in your API calls
+
+Example:
+```python
+from crawl4ai import CrawlerRunConfig, PruningContentFilter
+
+config = CrawlerRunConfig(
+    content_filter=PruningContentFilter(threshold=0.48)
+)
+print(config.dump())  # Use this JSON in your API calls
+```
+
+
+#### More Examples
+
+**Advanced Crawler Configuration**
+
+```json
+{
+    "urls": ["https://example.com"],
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "cache_mode": "bypass",
+            "markdown_generator": {
+                "type": "DefaultMarkdownGenerator",
+                "params": {
+                    "content_filter": {
+                        "type": "PruningContentFilter",
+                        "params": {
+                            "threshold": 0.48,
+                            "threshold_type": "fixed",
+                            "min_word_threshold": 0
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+**Extraction Strategy**:
+
+```json
+{
+    "crawler_config": {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "extraction_strategy": {
+                "type": "JsonCssExtractionStrategy",
+                "params": {
+                    "schema": {
+                        "baseSelector": "article.post",
+                        "fields": [
+                            {"name": "title", "selector": "h1", "type": "text"},
+                            {"name": "content", "selector": ".content", "type": "html"}
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+**LLM Extraction Strategy**
+
+```json
+{
+  "crawler_config": {
+    "type": "CrawlerRunConfig",
+    "params": {
+      "extraction_strategy": {
+        "type": "LLMExtractionStrategy",
+        "params": {
+          "instruction": "Extract article title, author, publication date and main content",
+          "provider": "openai/gpt-4",
+          "api_token": "your-api-token",
+          "schema": {
+            "type": "dict",
+            "value": {
+              "title": "Article Schema",
+              "type": "object",
+              "properties": {
+                "title": {
+                  "type": "string",
+                  "description": "The article's headline"
+                },
+                "author": {
+                  "type": "string",
+                  "description": "The author's name"
+                },
+                "published_date": {
+                  "type": "string",
+                  "format": "date-time",
+                  "description": "Publication date and time"
+                },
+                "content": {
+                  "type": "string",
+                  "description": "The main article content"
+                }
+              },
+              "required": ["title", "content"]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+**Deep Crawler Example**
+
+```json
+{
+  "crawler_config": {
+    "type": "CrawlerRunConfig",
+    "params": {
+      "deep_crawl_strategy": {
+        "type": "BFSDeepCrawlStrategy",
+        "params": {
+          "max_depth": 3,
+          "filter_chain": {
+            "type": "FilterChain",
+            "params": {
+              "filters": [
+                {
+                  "type": "ContentTypeFilter",
+                  "params": {
+                    "allowed_types": ["text/html", "application/xhtml+xml"]
+                  }
+                },
+                {
+                  "type": "DomainFilter",
+                  "params": {
+                    "allowed_domains": ["blog.*", "docs.*"],
+                  }
+                }
+              ]
+            }
+          },
+          "url_scorer": {
+            "type": "CompositeScorer",
+            "params": {
+              "scorers": [
+                {
+                  "type": "KeywordRelevanceScorer",
+                  "params": {
+                    "keywords": ["tutorial", "guide", "documentation"],
+                  }
+                },
+                {
+                  "type": "PathDepthScorer",
+                  "params": {
+                    "weight": 0.5,
+                    "optimal_depth": 3  
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+### REST API Examples
+
+Let's look at some practical examples:
+
+#### Simple Crawl
+
+```python
+import requests
+
+crawl_payload = {
+    "urls": ["https://example.com"],
+    "browser_config": {"headless": True},
+    "crawler_config": {"stream": False}
+}
+response = requests.post(
+    "http://localhost:8000/crawl",
+    # headers={"Authorization": f"Bearer {token}"},  # If JWT is enabled, more on this later
+    json=crawl_payload
+)
+print(response.json())  # Print the response for debugging
+```
+
+#### Streaming Results
+
+```python
+async def test_stream_crawl(session, token: str):
+    """Test the /crawl/stream endpoint with multiple URLs."""
+    url = "http://localhost:8000/crawl/stream"
+    payload = {
+        "urls": [
+            "https://example.com",
+            "https://example.com/page1",  
+            "https://example.com/page2",  
+            "https://example.com/page3",  
+        ],
+        "browser_config": {"headless": True, "viewport": {"width": 1200}},
+        "crawler_config": {"stream": True, "cache_mode": "aggressive"}
+    }
+
+    # headers = {"Authorization": f"Bearer {token}"} # If JWT is enabled, more on this later
+    
+    try:
+        async with session.post(url, json=payload, headers=headers) as response:
+            status = response.status
+            print(f"Status: {status} (Expected: 200)")
+            assert status == 200, f"Expected 200, got {status}"
+            
+            # Read streaming response line-by-line (NDJSON)
+            async for line in response.content:
+                if line:
+                    data = json.loads(line.decode('utf-8').strip())
+                    print(f"Streamed Result: {json.dumps(data, indent=2)}")
+    except Exception as e:
+        print(f"Error in streaming crawl test: {str(e)}")
+```
+
+## Metrics & Monitoring
+
+Keep an eye on your crawler with these endpoints:
+
+- `/health` - Quick health check
+- `/metrics` - Detailed Prometheus metrics
+- `/schema` - Full API schema
+
+Example health check:
+```bash
+curl http://localhost:8000/health
+```
+
+## Deployment Scenarios
+
+> 🚧 Coming soon! We'll cover:
+> - Kubernetes deployment
+> - Cloud provider setups (AWS, GCP, Azure)
+> - High-availability configurations
+> - Load balancing strategies
+
+## Complete Examples
+
+Check out the `examples` folder in our repository for full working examples! Here are two to get you started:
+[Using Client SDK](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_sdk_example.py)
+[Using REST API](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_python_rest_api_example.py)
+
+## Server Configuration
+
+The server's behavior can be customized through the `config.yml` file. Let's explore how to configure your Crawl4AI server for optimal performance and security.
+
+### Understanding config.yml
+
+The configuration file is located at `deploy/docker/config.yml`. You can either modify this file before building the image or mount a custom configuration when running the container.
+
+Here's a detailed breakdown of the configuration options:
+
+```yaml
+# Application Configuration
+app:
+  title: "Crawl4AI API"           # Server title in OpenAPI docs
+  version: "1.0.0"               # API version
+  host: "0.0.0.0"               # Listen on all interfaces
+  port: 8000                    # Server port
+  reload: True                  # Enable hot reloading (development only)
+  timeout_keep_alive: 300       # Keep-alive timeout in seconds
+
+# Rate Limiting Configuration
+rate_limiting:
+  enabled: True                 # Enable/disable rate limiting
+  default_limit: "100/minute"   # Rate limit format: "number/timeunit"
+  trusted_proxies: []          # List of trusted proxy IPs
+  storage_uri: "memory://"     # Use "redis://localhost:6379" for production
+
+# Security Configuration
+security:
+  enabled: false               # Master toggle for security features
+  jwt_enabled: true            # Enable JWT authentication
+  https_redirect: True         # Force HTTPS
+  trusted_hosts: ["*"]         # Allowed hosts (use specific domains in production)
+  headers:                     # Security headers
+    x_content_type_options: "nosniff"
+    x_frame_options: "DENY"
+    content_security_policy: "default-src 'self'"
+    strict_transport_security: "max-age=63072000; includeSubDomains"
+
+# Crawler Configuration
+crawler:
+  memory_threshold_percent: 95.0  # Memory usage threshold
+  rate_limiter:
+    base_delay: [1.0, 2.0]      # Min and max delay between requests
+  timeouts:
+    stream_init: 30.0           # Stream initialization timeout
+    batch_process: 300.0        # Batch processing timeout
+
+# Logging Configuration
+logging:
+  level: "INFO"                 # Log level (DEBUG, INFO, WARNING, ERROR)
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Observability Configuration
+observability:
+  prometheus:
+    enabled: True              # Enable Prometheus metrics
+    endpoint: "/metrics"       # Metrics endpoint
+  health_check:
+    endpoint: "/health"        # Health check endpoint
+```
+
+### JWT Authentication
+
+When `security.jwt_enabled` is set to `true` in your config.yml, all endpoints require JWT authentication via bearer tokens. Here's how it works:
+
+#### Getting a Token
+```python
+POST /token
+Content-Type: application/json
+
+{
+    "email": "user@example.com"
+}
+```
+
+The endpoint returns:
+```json
+{
+    "email": "user@example.com",
+    "access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOi...",
+    "token_type": "bearer"
+}
+```
+
+#### Using the Token
+Add the token to your requests:
+```bash
+curl -H "Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGci..." http://localhost:8000/crawl
+```
+
+Using the Python SDK:
+```python
+from crawl4ai.docker_client import Crawl4aiDockerClient
+
+async with Crawl4aiDockerClient() as client:
+    # Authenticate first
+    await client.authenticate("user@example.com")
+    
+    # Now all requests will include the token automatically
+    result = await client.crawl(urls=["https://example.com"])
+```
+
+#### Production Considerations 💡
+The default implementation uses a simple email verification. For production use, consider:
+- Email verification via OTP/magic links
+- OAuth2 integration
+- Rate limiting token generation
+- Token expiration and refresh mechanisms
+- IP-based restrictions
+
+### Configuration Tips and Best Practices
+
+1. **Production Settings** 🏭
+
+   ```yaml
+   app:
+     reload: False              # Disable reload in production
+     timeout_keep_alive: 120    # Lower timeout for better resource management
+   
+   rate_limiting:
+     storage_uri: "redis://redis:6379"  # Use Redis for distributed rate limiting
+     default_limit: "50/minute"         # More conservative rate limit
+   
+   security:
+     enabled: true                      # Enable all security features
+     trusted_hosts: ["your-domain.com"] # Restrict to your domain
+   ```
+
+2. **Development Settings** 🛠️
+
+   ```yaml
+   app:
+     reload: True               # Enable hot reloading
+     timeout_keep_alive: 300    # Longer timeout for debugging
+   
+   logging:
+     level: "DEBUG"            # More verbose logging
+   ```
+
+3. **High-Traffic Settings** 🚦
+
+   ```yaml
+   crawler:
+     memory_threshold_percent: 85.0  # More conservative memory limit
+     rate_limiter:
+       base_delay: [2.0, 4.0]       # More aggressive rate limiting
+   ```
+
+### Customizing Your Configuration
+
+#### Method 1: Pre-build Configuration
+
+```bash
+# Copy and modify config before building
+cd crawl4ai/deploy
+vim custom-config.yml # Or use any editor
+
+# Build with custom config
+docker build --platform=linux/amd64 --no-cache -t crawl4ai:latest .
+```
+
+#### Method 2: Build-time Configuration
+
+Use a custom config during build:
+
+```bash
+# Build with custom config
+docker build --platform=linux/amd64 --no-cache \
+  --build-arg CONFIG_PATH=/path/to/custom-config.yml \ 
+  -t crawl4ai:latest .
+```
+
+#### Method 3: Runtime Configuration
+```bash
+# Mount custom config at runtime
+docker run -d -p 8000:8000 \
+  -v $(pwd)/custom-config.yml:/app/config.yml \
+  crawl4ai-server:prod
+```
+
+> 💡 Note: When using Method 2, `/path/to/custom-config.yml` is relative to deploy directory.
+> 💡 Note: When using Method 3, ensure your custom config file has all required fields as the container will use this instead of the built-in config.
+
+### Configuration Recommendations
+
+1. **Security First** 🔒
+   - Always enable security in production
+   - Use specific trusted_hosts instead of wildcards
+   - Set up proper rate limiting to protect your server
+   - Consider your environment before enabling HTTPS redirect
+
+2. **Resource Management** 💻
+   - Adjust memory_threshold_percent based on available RAM
+   - Set timeouts according to your content size and network conditions
+   - Use Redis for rate limiting in multi-container setups
+
+3. **Monitoring** 📊
+   - Enable Prometheus if you need metrics
+   - Set DEBUG logging in development, INFO in production
+   - Regular health check monitoring is crucial
+
+4. **Performance Tuning** ⚡
+   - Start with conservative rate limiter delays
+   - Increase batch_process timeout for large content
+   - Adjust stream_init timeout based on initial response times
+
+## Getting Help
+
+We're here to help you succeed with Crawl4AI! Here's how to get support:
+
+- 📖 Check our [full documentation](https://docs.crawl4ai.com)
+- 🐛 Found a bug? [Open an issue](https://github.com/unclecode/crawl4ai/issues)
+- 💬 Join our [Discord community](https://discord.gg/crawl4ai)
+- ⭐ Star us on GitHub to show support!
+
+## Summary
+
+In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment:
+- Building and running the Docker container
+- Configuring the environment
+- Making API requests with proper typing
+- Using the Python SDK
+- Monitoring your deployment
+
+Remember, the examples in the `examples` folder are your friends - they show real-world usage patterns that you can adapt for your needs.
+
+Keep exploring, and don't hesitate to reach out if you need help! We're building something amazing together. 🚀
+
+Happy crawling! 🕷️

deploy/docker/api.py

@@ -0,0 +1,442 @@
+import os
+import json
+import asyncio
+from typing import List, Tuple
+
+import logging
+from typing import Optional, AsyncGenerator
+from urllib.parse import unquote
+from fastapi import HTTPException, Request, status
+from fastapi.background import BackgroundTasks
+from fastapi.responses import JSONResponse
+from redis import asyncio as aioredis
+
+from crawl4ai import (
+    AsyncWebCrawler,
+    CrawlerRunConfig,
+    LLMExtractionStrategy,
+    CacheMode,
+    BrowserConfig,
+    MemoryAdaptiveDispatcher,
+    RateLimiter
+)
+from crawl4ai.utils import perform_completion_with_backoff
+from crawl4ai.content_filter_strategy import (
+    PruningContentFilter,
+    BM25ContentFilter,
+    LLMContentFilter
+)
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+
+from utils import (
+    TaskStatus,
+    FilterType,
+    get_base_url,
+    is_task_id,
+    should_cleanup_task,
+    decode_redis_hash
+)
+
+logger = logging.getLogger(__name__)
+
+async def handle_llm_qa(
+    url: str,
+    query: str,
+    config: dict
+) -> str:
+    """Process QA using LLM with crawled content as context."""
+    try:
+        # Extract base URL by finding last '?q=' occurrence
+        last_q_index = url.rfind('?q=')
+        if last_q_index != -1:
+            url = url[:last_q_index]
+
+        # Get markdown content
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(url)
+            if not result.success:
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=result.error_message
+                )
+            content = result.markdown.fit_markdown
+
+        # Create prompt and get LLM response
+        prompt = f"""Use the following content as context to answer the question.
+    Content:
+    {content}
+
+    Question: {query}
+
+    Answer:"""
+
+        response = perform_completion_with_backoff(
+            provider=config["llm"]["provider"],
+            prompt_with_variables=prompt,
+            api_token=os.environ.get(config["llm"].get("api_key_env", ""))
+        )
+
+        return response.choices[0].message.content
+    except Exception as e:
+        logger.error(f"QA processing error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def process_llm_extraction(
+    redis: aioredis.Redis,
+    config: dict,
+    task_id: str,
+    url: str,
+    instruction: str,
+    schema: Optional[str] = None,
+    cache: str = "0"
+) -> None:
+    """Process LLM extraction in background."""
+    try:
+        # If config['llm'] has api_key then ignore the api_key_env
+        api_key = ""
+        if "api_key" in config["llm"]:
+            api_key = config["llm"]["api_key"]
+        else:
+            api_key = os.environ.get(config["llm"].get("api_key_env", None), "")
+        llm_strategy = LLMExtractionStrategy(
+            provider=config["llm"]["provider"],
+            api_token=api_key,
+            instruction=instruction,
+            schema=json.loads(schema) if schema else None,
+        )
+
+        cache_mode = CacheMode.ENABLED if cache == "1" else CacheMode.WRITE_ONLY
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=url,
+                config=CrawlerRunConfig(
+                    extraction_strategy=llm_strategy,
+                    scraping_strategy=LXMLWebScrapingStrategy(),
+                    cache_mode=cache_mode
+                )
+            )
+
+        if not result.success:
+            await redis.hset(f"task:{task_id}", mapping={
+                "status": TaskStatus.FAILED,
+                "error": result.error_message
+            })
+            return
+
+        try:
+            content = json.loads(result.extracted_content)
+        except json.JSONDecodeError:
+            content = result.extracted_content
+        await redis.hset(f"task:{task_id}", mapping={
+            "status": TaskStatus.COMPLETED,
+            "result": json.dumps(content)
+        })
+
+    except Exception as e:
+        logger.error(f"LLM extraction error: {str(e)}", exc_info=True)
+        await redis.hset(f"task:{task_id}", mapping={
+            "status": TaskStatus.FAILED,
+            "error": str(e)
+        })
+
+async def handle_markdown_request(
+    url: str,
+    filter_type: FilterType,
+    query: Optional[str] = None,
+    cache: str = "0",
+    config: Optional[dict] = None
+) -> str:
+    """Handle markdown generation requests."""
+    try:
+        decoded_url = unquote(url)
+        if not decoded_url.startswith(('http://', 'https://')):
+            decoded_url = 'https://' + decoded_url
+
+        if filter_type == FilterType.RAW:
+            md_generator = DefaultMarkdownGenerator()
+        else:
+            content_filter = {
+                FilterType.FIT: PruningContentFilter(),
+                FilterType.BM25: BM25ContentFilter(user_query=query or ""),
+                FilterType.LLM: LLMContentFilter(
+                    provider=config["llm"]["provider"],
+                    api_token=os.environ.get(config["llm"].get("api_key_env", None), ""),
+                    instruction=query or "Extract main content"
+                )
+            }[filter_type]
+            md_generator = DefaultMarkdownGenerator(content_filter=content_filter)
+
+        cache_mode = CacheMode.ENABLED if cache == "1" else CacheMode.WRITE_ONLY
+
+        async with AsyncWebCrawler() as crawler:
+            result = await crawler.arun(
+                url=decoded_url,
+                config=CrawlerRunConfig(
+                    markdown_generator=md_generator,
+                    scraping_strategy=LXMLWebScrapingStrategy(),
+                    cache_mode=cache_mode
+                )
+            )
+            
+            if not result.success:
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=result.error_message
+                )
+
+            return (result.markdown.raw_markdown 
+                   if filter_type == FilterType.RAW 
+                   else result.markdown.fit_markdown)
+
+    except Exception as e:
+        logger.error(f"Markdown error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def handle_llm_request(
+    redis: aioredis.Redis,
+    background_tasks: BackgroundTasks,
+    request: Request,
+    input_path: str,
+    query: Optional[str] = None,
+    schema: Optional[str] = None,
+    cache: str = "0",
+    config: Optional[dict] = None
+) -> JSONResponse:
+    """Handle LLM extraction requests."""
+    base_url = get_base_url(request)
+    
+    try:
+        if is_task_id(input_path):
+            return await handle_task_status(
+                redis, input_path, base_url
+            )
+
+        if not query:
+            return JSONResponse({
+                "message": "Please provide an instruction",
+                "_links": {
+                    "example": {
+                        "href": f"{base_url}/llm/{input_path}?q=Extract+main+content",
+                        "title": "Try this example"
+                    }
+                }
+            })
+
+        return await create_new_task(
+            redis,
+            background_tasks,
+            input_path,
+            query,
+            schema,
+            cache,
+            base_url,
+            config
+        )
+
+    except Exception as e:
+        logger.error(f"LLM endpoint error: {str(e)}", exc_info=True)
+        return JSONResponse({
+            "error": str(e),
+            "_links": {
+                "retry": {"href": str(request.url)}
+            }
+        }, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+async def handle_task_status(
+    redis: aioredis.Redis,
+    task_id: str,
+    base_url: str
+) -> JSONResponse:
+    """Handle task status check requests."""
+    task = await redis.hgetall(f"task:{task_id}")
+    if not task:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Task not found"
+        )
+
+    task = decode_redis_hash(task)
+    response = create_task_response(task, task_id, base_url)
+
+    if task["status"] in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
+        if should_cleanup_task(task["created_at"]):
+            await redis.delete(f"task:{task_id}")
+
+    return JSONResponse(response)
+
+async def create_new_task(
+    redis: aioredis.Redis,
+    background_tasks: BackgroundTasks,
+    input_path: str,
+    query: str,
+    schema: Optional[str],
+    cache: str,
+    base_url: str,
+    config: dict
+) -> JSONResponse:
+    """Create and initialize a new task."""
+    decoded_url = unquote(input_path)
+    if not decoded_url.startswith(('http://', 'https://')):
+        decoded_url = 'https://' + decoded_url
+
+    from datetime import datetime
+    task_id = f"llm_{int(datetime.now().timestamp())}_{id(background_tasks)}"
+    
+    await redis.hset(f"task:{task_id}", mapping={
+        "status": TaskStatus.PROCESSING,
+        "created_at": datetime.now().isoformat(),
+        "url": decoded_url
+    })
+
+    background_tasks.add_task(
+        process_llm_extraction,
+        redis,
+        config,
+        task_id,
+        decoded_url,
+        query,
+        schema,
+        cache
+    )
+
+    return JSONResponse({
+        "task_id": task_id,
+        "status": TaskStatus.PROCESSING,
+        "url": decoded_url,
+        "_links": {
+            "self": {"href": f"{base_url}/llm/{task_id}"},
+            "status": {"href": f"{base_url}/llm/{task_id}"}
+        }
+    })
+
+def create_task_response(task: dict, task_id: str, base_url: str) -> dict:
+    """Create response for task status check."""
+    response = {
+        "task_id": task_id,
+        "status": task["status"],
+        "created_at": task["created_at"],
+        "url": task["url"],
+        "_links": {
+            "self": {"href": f"{base_url}/llm/{task_id}"},
+            "refresh": {"href": f"{base_url}/llm/{task_id}"}
+        }
+    }
+
+    if task["status"] == TaskStatus.COMPLETED:
+        response["result"] = json.loads(task["result"])
+    elif task["status"] == TaskStatus.FAILED:
+        response["error"] = task["error"]
+
+    return response
+
+async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) -> AsyncGenerator[bytes, None]:
+    """Stream results with heartbeats and completion markers."""
+    import json
+    from utils import datetime_handler
+
+    try:
+        async for result in results_gen:
+            try:
+                result_dict = result.model_dump()
+                logger.info(f"Streaming result for {result_dict.get('url', 'unknown')}")
+                data = json.dumps(result_dict, default=datetime_handler) + "\n"
+                yield data.encode('utf-8')
+            except Exception as e:
+                logger.error(f"Serialization error: {e}")
+                error_response = {"error": str(e), "url": getattr(result, 'url', 'unknown')}
+                yield (json.dumps(error_response) + "\n").encode('utf-8')
+
+        yield json.dumps({"status": "completed"}).encode('utf-8')
+        
+    except asyncio.CancelledError:
+        logger.warning("Client disconnected during streaming")
+    finally:
+        try:
+            await crawler.close()
+        except Exception as e:
+            logger.error(f"Crawler cleanup error: {e}")
+
+async def handle_crawl_request(
+    urls: List[str],
+    browser_config: dict,
+    crawler_config: dict,
+    config: dict
+) -> dict:
+    """Handle non-streaming crawl requests."""
+    try:
+        browser_config = BrowserConfig.load(browser_config)
+        crawler_config = CrawlerRunConfig.load(crawler_config)
+
+        dispatcher = MemoryAdaptiveDispatcher(
+            memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
+            rate_limiter=RateLimiter(
+                base_delay=tuple(config["crawler"]["rate_limiter"]["base_delay"])
+            )
+        )
+
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            results = await crawler.arun_many(
+                urls=urls,
+                config=crawler_config,
+                dispatcher=dispatcher
+            )
+            
+            return {
+                "success": True,
+                "results": [result.model_dump() for result in results]
+            }
+
+    except Exception as e:
+        logger.error(f"Crawl error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )
+
+async def handle_stream_crawl_request(
+    urls: List[str],
+    browser_config: dict,
+    crawler_config: dict,
+    config: dict
+) -> Tuple[AsyncWebCrawler, AsyncGenerator]:
+    """Handle streaming crawl requests."""
+    try:
+        browser_config = BrowserConfig.load(browser_config)
+        browser_config.verbose = True
+        crawler_config = CrawlerRunConfig.load(crawler_config)
+        crawler_config.scraping_strategy = LXMLWebScrapingStrategy()
+
+        dispatcher = MemoryAdaptiveDispatcher(
+            memory_threshold_percent=config["crawler"]["memory_threshold_percent"],
+            rate_limiter=RateLimiter(
+                base_delay=tuple(config["crawler"]["rate_limiter"]["base_delay"])
+            )
+        )
+
+        crawler = AsyncWebCrawler(config=browser_config)
+        await crawler.start()
+
+        results_gen = await crawler.arun_many(
+            urls=urls,
+            config=crawler_config,
+            dispatcher=dispatcher
+        )
+
+        return crawler, results_gen
+
+    except Exception as e:
+        if 'crawler' in locals():
+            await crawler.close()
+        logger.error(f"Stream crawl error: {str(e)}", exc_info=True)
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )

deploy/docker/auth.py

@@ -0,0 +1,46 @@
+import os
+from datetime import datetime, timedelta, timezone
+from typing import Dict, Optional
+from jwt import JWT, jwk_from_dict
+from jwt.utils import get_int_from_datetime
+from fastapi import Depends, HTTPException
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from pydantic import EmailStr
+from pydantic.main import BaseModel
+import base64
+
+instance = JWT()
+security = HTTPBearer()
+SECRET_KEY = os.environ.get("SECRET_KEY", "mysecret")
+ACCESS_TOKEN_EXPIRE_MINUTES = 60
+
+def get_jwk_from_secret(secret: str):
+    """Convert a secret string into a JWK object."""
+    secret_bytes = secret.encode('utf-8')
+    b64_secret = base64.urlsafe_b64encode(secret_bytes).rstrip(b'=').decode('utf-8')
+    return jwk_from_dict({"kty": "oct", "k": b64_secret})
+
+def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str:
+    """Create a JWT access token with an expiration."""
+    to_encode = data.copy()
+    expire = datetime.now(timezone.utc) + (expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES))
+    to_encode.update({"exp": get_int_from_datetime(expire)})
+    signing_key = get_jwk_from_secret(SECRET_KEY)
+    return instance.encode(to_encode, signing_key, alg='HS256')
+
+def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)) -> Dict:
+    """Verify the JWT token from the Authorization header."""
+    token = credentials.credentials
+    verifying_key = get_jwk_from_secret(SECRET_KEY)
+    try:
+        payload = instance.decode(token, verifying_key, do_time_check=True, algorithms='HS256')
+        return payload
+    except Exception:
+        raise HTTPException(status_code=401, detail="Invalid or expired token")
+
+def get_token_dependency(config: Dict):
+    """Return the token dependency if JWT is enabled, else None."""
+    return verify_token if config.get("security", {}).get("jwt_enabled", False) else None
+
+class TokenRequest(BaseModel):
+    email: EmailStr

deploy/docker/config.yml

@@ -0,0 +1,71 @@
+# Application Configuration
+app:
+  title: "Crawl4AI API"
+  version: "1.0.0"
+  host: "0.0.0.0"
+  port: 8000
+  reload: True
+  timeout_keep_alive: 300
+
+# Default LLM Configuration
+llm:
+  provider: "openai/gpt-4o-mini"
+  api_key_env: "OPENAI_API_KEY"
+  # api_key: sk-...  # If you pass the API key directly then api_key_env will be ignored
+
+# Redis Configuration
+redis:
+  host: "localhost"
+  port: 6379
+  db: 0
+  password: ""
+  ssl: False
+  ssl_cert_reqs: None
+  ssl_ca_certs: None
+  ssl_certfile: None
+  ssl_keyfile: None
+  ssl_cert_reqs: None
+  ssl_ca_certs: None
+  ssl_certfile: None
+  ssl_keyfile: None
+
+# Rate Limiting Configuration
+rate_limiting:
+  enabled: True
+  default_limit: "1000/minute"
+  trusted_proxies: []
+  storage_uri: "memory://"  # Use "redis://localhost:6379" for production
+
+# Security Configuration
+security:
+  enabled: true 
+  jwt_enabled: true 
+  https_redirect: false
+  trusted_hosts: ["*"]
+  headers:
+    x_content_type_options: "nosniff"
+    x_frame_options: "DENY"
+    content_security_policy: "default-src 'self'"
+    strict_transport_security: "max-age=63072000; includeSubDomains"
+
+# Crawler Configuration
+crawler:
+  memory_threshold_percent: 95.0
+  rate_limiter:
+    base_delay: [1.0, 2.0]
+  timeouts:
+    stream_init: 30.0  # Timeout for stream initialization
+    batch_process: 300.0  # Timeout for batch processing
+
+# Logging Configuration
+logging:
+  level: "INFO"
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+# Observability Configuration
+observability:
+  prometheus:
+    enabled: True
+    endpoint: "/metrics"
+  health_check:
+    endpoint: "/health"

deploy/docker/requirements.txt

@@ -0,0 +1,10 @@
+crawl4ai
+fastapi
+uvicorn
+gunicorn>=23.0.0
+slowapi>=0.1.9
+prometheus-fastapi-instrumentator>=7.0.2
+redis>=5.2.1
+jwt>=1.3.1
+dnspython>=2.7.0
+email-validator>=2.2.0

deploy/docker/server.py

@@ -0,0 +1,181 @@
+import os
+import sys
+import time
+from typing import List, Optional, Dict
+from fastapi import FastAPI, HTTPException, Request, Query, Path, Depends
+from fastapi.responses import StreamingResponse, RedirectResponse, PlainTextResponse, JSONResponse
+from fastapi.middleware.httpsredirect import HTTPSRedirectMiddleware
+from fastapi.middleware.trustedhost import TrustedHostMiddleware
+from pydantic import BaseModel, Field
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+from prometheus_fastapi_instrumentator import Instrumentator
+from redis import asyncio as aioredis
+
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+from utils import FilterType, load_config, setup_logging, verify_email_domain
+from api import (
+    handle_markdown_request,
+    handle_llm_qa,
+    handle_stream_crawl_request,
+    handle_crawl_request,
+    stream_results
+)
+from auth import create_access_token, get_token_dependency, TokenRequest  # Import from auth.py
+
+__version__ = "0.2.6"
+
+class CrawlRequest(BaseModel):
+    urls: List[str] = Field(min_length=1, max_length=100)
+    browser_config: Optional[Dict] = Field(default_factory=dict)
+    crawler_config: Optional[Dict] = Field(default_factory=dict)
+
+# Load configuration and setup
+config = load_config()
+setup_logging(config)
+
+# Initialize Redis
+redis = aioredis.from_url(config["redis"].get("uri", "redis://localhost"))
+
+# Initialize rate limiter
+limiter = Limiter(
+    key_func=get_remote_address,
+    default_limits=[config["rate_limiting"]["default_limit"]],
+    storage_uri=config["rate_limiting"]["storage_uri"]
+)
+
+app = FastAPI(
+    title=config["app"]["title"],
+    version=config["app"]["version"]
+)
+
+# Configure middleware
+def setup_security_middleware(app, config):
+    sec_config = config.get("security", {})
+    if sec_config.get("enabled", False):
+        if sec_config.get("https_redirect", False):
+            app.add_middleware(HTTPSRedirectMiddleware)
+        if sec_config.get("trusted_hosts", []) != ["*"]:
+            app.add_middleware(TrustedHostMiddleware, allowed_hosts=sec_config["trusted_hosts"])
+
+setup_security_middleware(app, config)
+
+# Prometheus instrumentation
+if config["observability"]["prometheus"]["enabled"]:
+    Instrumentator().instrument(app).expose(app)
+
+# Get token dependency based on config
+token_dependency = get_token_dependency(config)
+
+# Middleware for security headers
+@app.middleware("http")
+async def add_security_headers(request: Request, call_next):
+    response = await call_next(request)
+    if config["security"]["enabled"]:
+        response.headers.update(config["security"]["headers"])
+    return response
+
+# Token endpoint (always available, but usage depends on config)
+@app.post("/token")
+async def get_token(request_data: TokenRequest):
+    if not verify_email_domain(request_data.email):
+        raise HTTPException(status_code=400, detail="Invalid email domain")
+    token = create_access_token({"sub": request_data.email})
+    return {"email": request_data.email, "access_token": token, "token_type": "bearer"}
+
+# Endpoints with conditional auth
+@app.get("/md/{url:path}")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def get_markdown(
+    request: Request,
+    url: str,
+    f: FilterType = FilterType.FIT,
+    q: Optional[str] = None,
+    c: Optional[str] = "0",
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    result = await handle_markdown_request(url, f, q, c, config)
+    return PlainTextResponse(result)
+
+@app.get("/llm/{url:path}", description="URL should be without http/https prefix")
+async def llm_endpoint(
+    request: Request,
+    url: str = Path(...),
+    q: Optional[str] = Query(None),
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not q:
+        raise HTTPException(status_code=400, detail="Query parameter 'q' is required")
+    if not url.startswith(('http://', 'https://')):
+        url = 'https://' + url
+    try:
+        answer = await handle_llm_qa(url, q, config)
+        return JSONResponse({"answer": answer})
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/schema")
+async def get_schema():
+    from crawl4ai import BrowserConfig, CrawlerRunConfig
+    return {"browser": BrowserConfig().dump(), "crawler": CrawlerRunConfig().dump()}
+
+@app.get(config["observability"]["health_check"]["endpoint"])
+async def health():
+    return {"status": "ok", "timestamp": time.time(), "version": __version__}
+
+@app.get(config["observability"]["prometheus"]["endpoint"])
+async def metrics():
+    return RedirectResponse(url=config["observability"]["prometheus"]["endpoint"])
+
+@app.post("/crawl")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def crawl(
+    request: Request,
+    crawl_request: CrawlRequest,
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not crawl_request.urls:
+        raise HTTPException(status_code=400, detail="At least one URL required")
+    
+    results = await handle_crawl_request(
+        urls=crawl_request.urls,
+        browser_config=crawl_request.browser_config,
+        crawler_config=crawl_request.crawler_config,
+        config=config
+    )
+
+    return JSONResponse(results)
+
+
+@app.post("/crawl/stream")
+@limiter.limit(config["rate_limiting"]["default_limit"])
+async def crawl_stream(
+    request: Request,
+    crawl_request: CrawlRequest,
+    token_data: Optional[Dict] = Depends(token_dependency)
+):
+    if not crawl_request.urls:
+        raise HTTPException(status_code=400, detail="At least one URL required")
+
+    crawler, results_gen = await handle_stream_crawl_request(
+        urls=crawl_request.urls,
+        browser_config=crawl_request.browser_config,
+        crawler_config=crawl_request.crawler_config,
+        config=config
+    )
+
+    return StreamingResponse(
+        stream_results(crawler, results_gen),
+        media_type='application/x-ndjson',
+        headers={'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'X-Stream-Status': 'active'}
+    )
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(
+        "server:app",
+        host=config["app"]["host"],
+        port=config["app"]["port"],
+        reload=config["app"]["reload"],
+        timeout_keep_alive=config["app"]["timeout_keep_alive"]
+    )

deploy/docker/supervisord.conf

@@ -0,0 +1,12 @@
+[supervisord]
+nodaemon=true
+
+[program:redis]
+command=redis-server
+autorestart=true
+priority=10
+
+[program:gunicorn]
+command=gunicorn --bind 0.0.0.0:8000 --workers 4 --threads 2 --timeout 300 --graceful-timeout 60 --keep-alive 65 --log-level debug --worker-class uvicorn.workers.UvicornWorker --max-requests 1000 --max-requests-jitter 50 server:app
+autorestart=true
+priority=20

deploy/docker/utils.py

@@ -0,0 +1,66 @@
+import dns.resolver
+import logging
+import yaml
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from fastapi import Request
+from typing import Dict, Optional
+
+class TaskStatus(str, Enum):
+    PROCESSING = "processing"
+    FAILED = "failed"
+    COMPLETED = "completed"
+
+class FilterType(str, Enum):
+    RAW = "raw"
+    FIT = "fit"
+    BM25 = "bm25"
+    LLM = "llm"
+
+def load_config() -> Dict:
+    """Load and return application configuration."""
+    config_path = Path(__file__).parent / "config.yml"
+    with open(config_path, "r") as config_file:
+        return yaml.safe_load(config_file)
+
+def setup_logging(config: Dict) -> None:
+    """Configure application logging."""
+    logging.basicConfig(
+        level=config["logging"]["level"],
+        format=config["logging"]["format"]
+    )
+
+def get_base_url(request: Request) -> str:
+    """Get base URL including scheme and host."""
+    return f"{request.url.scheme}://{request.url.netloc}"
+
+def is_task_id(value: str) -> bool:
+    """Check if the value matches task ID pattern."""
+    return value.startswith("llm_") and "_" in value
+
+def datetime_handler(obj: any) -> Optional[str]:
+    """Handle datetime serialization for JSON."""
+    if hasattr(obj, 'isoformat'):
+        return obj.isoformat()
+    raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
+
+def should_cleanup_task(created_at: str) -> bool:
+    """Check if task should be cleaned up based on creation time."""
+    created = datetime.fromisoformat(created_at)
+    return (datetime.now() - created).total_seconds() > 3600
+
+def decode_redis_hash(hash_data: Dict[bytes, bytes]) -> Dict[str, str]:
+    """Decode Redis hash data from bytes to strings."""
+    return {k.decode('utf-8'): v.decode('utf-8') for k, v in hash_data.items()}
+
+
+
+def verify_email_domain(email: str) -> bool:
+    try:
+        domain = email.split('@')[1]
+        # Try to resolve MX records for the domain.
+        records = dns.resolver.resolve(domain, 'MX')
+        return True if records else False
+    except Exception as e:
+        return False

docker-compose.yml

@@ -1,5 +1,33 @@
+# Base configuration (not a service, just a reusable config block)
+x-base-config: &base-config
+  ports:
+    - "11235:11235"
+    - "8000:8000"
+    - "9222:9222"
+    - "8080:8080"
+  environment:
+    - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
+    - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+    - CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
+  volumes:
+    - /dev/shm:/dev/shm
+  deploy:
+    resources:
+      limits:
+        memory: 4G
+      reservations:
+        memory: 1G
+  restart: unless-stopped
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
+    interval: 30s
+    timeout: 10s
+    retries: 3
+    start_period: 40s
+
 services:
-  crawl4ai:
+  # Local build services for different platforms
+  crawl4ai-amd64:
     build:
       context: .
       dockerfile: Dockerfile
@@ -7,56 +35,31 @@ services:
         PYTHON_VERSION: "3.10"
         INSTALL_TYPE: ${INSTALL_TYPE:-basic}
         ENABLE_GPU: false
-    profiles: ["local"]
-    ports:
-      - "11235:11235"
-      - "8000:8000"
-      - "9222:9222"
-      - "8080:8080"
-    environment:
-      - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
-      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
-      - CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
-    volumes:
-      - /dev/shm:/dev/shm
-    deploy:
-      resources:
-        limits:
-          memory: 4G
-        reservations:
-          memory: 1G
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 40s
+      platforms:
+        - linux/amd64
+    profiles: ["local-amd64"]
+    <<: *base-config  # extends yerine doğrudan yapılandırmayı dahil ettik
 
-  crawl4ai-hub:
-    image: unclecode/crawl4ai:basic
-    profiles: ["hub"]
-    ports:
-      - "11235:11235"
-      - "8000:8000"
-      - "9222:9222"
-      - "8080:8080"
-    environment:
-      - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
-      - OPENAI_API_KEY=${OPENAI_API_KEY:-}
-      - CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
-    volumes:
-      - /dev/shm:/dev/shm
-    deploy:
-      resources:
-        limits:
-          memory: 4G
-        reservations:
-          memory: 1G
-    restart: unless-stopped
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 40s
+  crawl4ai-arm64:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        PYTHON_VERSION: "3.10"
+        INSTALL_TYPE: ${INSTALL_TYPE:-basic}
+        ENABLE_GPU: false
+      platforms:
+        - linux/arm64
+    profiles: ["local-arm64"]
+    <<: *base-config
+
+  # Hub services for different platforms and versions
+  crawl4ai-hub-amd64:
+    image: unclecode/crawl4ai:${VERSION:-basic}-amd64
+    profiles: ["hub-amd64"]
+    <<: *base-config
+
+  crawl4ai-hub-arm64:
+    image: unclecode/crawl4ai:${VERSION:-basic}-arm64
+    profiles: ["hub-arm64"]
+    <<: *base-config

docs/assets/powered-by-dark.svg

@@ -0,0 +1,25 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="120" height="35" viewBox="0 0 120 35">
+  <!-- Dark Theme -->
+  <g>
+    <defs>
+      <pattern id="halftoneDark" width="4" height="4" patternUnits="userSpaceOnUse">
+        <circle cx="2" cy="2" r="1" fill="#eee" opacity="0.1"/>
+      </pattern>
+      <pattern id="halftoneTextDark" width="3" height="3" patternUnits="userSpaceOnUse">
+        <circle cx="1.5" cy="1.5" r="2" fill="#aaa" opacity="0.2"/>
+      </pattern>
+    </defs>
+    <!-- White border - added as outer rectangle -->
+    <rect width="120" height="35" rx="5" fill="#111"/>
+    <!-- Dark background slightly smaller to show thicker border -->
+    <rect x="2" y="2" width="116" height="31" rx="4" fill="#1a1a1a"/>
+    <rect x="2" y="2" width="116" height="31" rx="4" fill="url(#halftoneDark)"/>
+    
+    <!-- Logo with halftone -->
+    <path d="M30 17.5 a7.5 7.5 0 1 1 -15 0 a7.5 7.5 0 1 1 15 0" fill="none" stroke="#eee" stroke-width="2"/>
+    <path d="M18 17.5 L27 17.5" stroke="#eee" stroke-width="2"/>
+    <circle cx="22.5" cy="17.5" r="2" fill="#eee"/>
+    
+    <text x="40" y="23" fill="#eee" font-family="Arial, sans-serif" font-weight="500" font-size="14">Crawl4AI</text>
+  </g>
+</svg>

docs/assets/powered-by-disco.svg

@@ -0,0 +1,64 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="120" height="35" viewBox="0 0 120 35">
+  <g>
+    <defs>
+      <pattern id="cyberdots" width="4" height="4" patternUnits="userSpaceOnUse">
+        <circle cx="2" cy="2" r="1">
+          <animate attributeName="fill" 
+                   values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4" 
+                   dur="6s" 
+                   repeatCount="indefinite"/>
+          <animate attributeName="opacity" 
+                   values="0.2;0.4;0.2" 
+                   dur="4s" 
+                   repeatCount="indefinite"/>
+        </circle>
+      </pattern>
+      <filter id="neonGlow" x="-20%" y="-20%" width="140%" height="140%">
+        <feGaussianBlur stdDeviation="1" result="blur"/>
+        <feFlood flood-color="#FF2EC4" flood-opacity="0.2">
+          <animate attributeName="flood-color"
+                   values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
+                   dur="8s"
+                   repeatCount="indefinite"/>
+        </feFlood>
+        <feComposite in2="blur" operator="in"/>
+        <feMerge>
+          <feMergeNode/>
+          <feMergeNode in="SourceGraphic"/>
+        </feMerge>
+      </filter>
+    </defs>
+    
+    <rect width="120" height="35" rx="5" fill="#0A0A0F"/>
+    <rect x="2" y="2" width="116" height="31" rx="4" fill="#16161E"/>
+    <rect x="2" y="2" width="116" height="31" rx="4" fill="url(#cyberdots)"/>
+    
+    <!-- Logo with animated neon -->
+    <path d="M30 17.5 a7.5 7.5 0 1 1 -15 0 a7.5 7.5 0 1 1 15 0" fill="none" stroke="#8B5CF6" stroke-width="2" filter="url(#neonGlow)">
+      <animate attributeName="stroke"
+               values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
+               dur="8s"
+               repeatCount="indefinite"/>
+    </path>
+    <path d="M18 17.5 L27 17.5" stroke="#8B5CF6" stroke-width="2" filter="url(#neonGlow)">
+      <animate attributeName="stroke"
+               values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
+               dur="8s"
+               repeatCount="indefinite"/>
+    </path>
+    <circle cx="22.5" cy="17.5" r="2" fill="#0BC5EA">
+      <animate attributeName="fill" 
+               values="#0BC5EA;#FF2EC4;#8B5CF6;#0BC5EA" 
+               dur="8s" 
+               repeatCount="indefinite"/>
+    </circle>
+    
+    <text x="40" y="23" font-family="Arial, sans-serif" font-weight="500" font-size="14" filter="url(#neonGlow)">
+      <animate attributeName="fill"
+               values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
+               dur="8s"
+               repeatCount="indefinite"/>
+      Crawl4AI
+    </text>
+  </g>
+</svg>

docs/assets/powered-by-light.svg

@@ -0,0 +1,21 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="120" height="35" viewBox="0 0 120 35">
+  <g>
+    <defs>
+      <pattern id="halftoneLight" width="4" height="4" patternUnits="userSpaceOnUse">
+        <circle cx="2" cy="2" r="1" fill="#111" opacity="0.1"/>
+      </pattern>
+    </defs>
+    <!-- Dark border -->
+    <rect width="120" height="35" rx="5" fill="#DDD"/>
+    <!-- Light background -->
+    <rect x="2" y="2" width="116" height="31" rx="4" fill="#fff"/>
+    <rect x="2" y="2" width="116" height="31" rx="4" fill="url(#halftoneLight)"/>
+    
+    <!-- Logo -->
+    <path d="M30 17.5 a7.5 7.5 0 1 1 -15 0 a7.5 7.5 0 1 1 15 0" fill="none" stroke="#111" stroke-width="2"/>
+    <path d="M18 17.5 L27 17.5" stroke="#111" stroke-width="2"/>
+    <circle cx="22.5" cy="17.5" r="2" fill="#111"/>
+    
+    <text x="40" y="23" fill="#111" font-family="Arial, sans-serif" font-weight="500" font-size="14">Crawl4AI</text>
+  </g>
+</svg>

docs/assets/powered-by-night.svg

@@ -0,0 +1,28 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="120" height="35" viewBox="0 0 120 35">
+  <g>
+    <defs>
+      <pattern id="halftoneDark" width="4" height="4" patternUnits="userSpaceOnUse">
+        <circle cx="2" cy="2" r="1" fill="#8B5CF6" opacity="0.1"/>
+      </pattern>
+      <filter id="neonGlow" x="-20%" y="-20%" width="140%" height="140%">
+        <feGaussianBlur stdDeviation="1" result="blur"/>
+        <feFlood flood-color="#8B5CF6" flood-opacity="0.2"/>
+        <feComposite in2="blur" operator="in"/>
+        <feMerge>
+          <feMergeNode/>
+          <feMergeNode in="SourceGraphic"/>
+        </feMerge>
+      </filter>
+    </defs>
+    <rect width="120" height="35" rx="5" fill="#0A0A0F"/>
+    <rect x="2" y="2" width="116" height="31" rx="4" fill="#16161E"/>
+    <rect x="2" y="2" width="116" height="31" rx="4" fill="url(#halftoneDark)"/>
+    
+    <!-- Logo with neon glow -->
+    <path d="M30 17.5 a7.5 7.5 0 1 1 -15 0 a7.5 7.5 0 1 1 15 0" fill="none" stroke="#8B5CF6" stroke-width="2" filter="url(#neonGlow)"/>
+    <path d="M18 17.5 L27 17.5" stroke="#8B5CF6" stroke-width="2" filter="url(#neonGlow)"/>
+    <circle cx="22.5" cy="17.5" r="2" fill="#8B5CF6"/>
+    
+    <text x="40" y="23" fill="#fff" font-family="Arial, sans-serif" font-weight="500" font-size="14" filter="url(#neonGlow)">Crawl4AI</text>
+  </g>
+</svg>

docs/deprecated/docker-deployment.md

@@ -0,0 +1,189 @@
+# 🐳 Using Docker (Legacy)
+
+Crawl4AI is available as Docker images for easy deployment. You can either pull directly from Docker Hub (recommended) or build from the repository.
+
+---
+
+<details>
+<summary>🐳 <strong>Option 1: Docker Hub (Recommended)</strong></summary>
+
+Choose the appropriate image based on your platform and needs:
+
+### For AMD64 (Regular Linux/Windows):
+```bash
+# Basic version (recommended)
+docker pull unclecode/crawl4ai:basic-amd64
+docker run -p 11235:11235 unclecode/crawl4ai:basic-amd64
+
+# Full ML/LLM support
+docker pull unclecode/crawl4ai:all-amd64
+docker run -p 11235:11235 unclecode/crawl4ai:all-amd64
+
+# With GPU support
+docker pull unclecode/crawl4ai:gpu-amd64
+docker run -p 11235:11235 unclecode/crawl4ai:gpu-amd64
+```
+
+### For ARM64 (M1/M2 Macs, ARM servers):
+```bash
+# Basic version (recommended)
+docker pull unclecode/crawl4ai:basic-arm64
+docker run -p 11235:11235 unclecode/crawl4ai:basic-arm64
+
+# Full ML/LLM support
+docker pull unclecode/crawl4ai:all-arm64
+docker run -p 11235:11235 unclecode/crawl4ai:all-arm64
+
+# With GPU support
+docker pull unclecode/crawl4ai:gpu-arm64
+docker run -p 11235:11235 unclecode/crawl4ai:gpu-arm64
+```
+
+Need more memory? Add `--shm-size`:
+```bash
+docker run --shm-size=2gb -p 11235:11235 unclecode/crawl4ai:basic-amd64
+```
+
+Test the installation:
+```bash
+curl http://localhost:11235/health
+```
+
+### For Raspberry Pi (32-bit) (coming soon):
+```bash
+# Pull and run basic version (recommended for Raspberry Pi)
+docker pull unclecode/crawl4ai:basic-armv7
+docker run -p 11235:11235 unclecode/crawl4ai:basic-armv7
+
+# With increased shared memory if needed
+docker run --shm-size=2gb -p 11235:11235 unclecode/crawl4ai:basic-armv7
+```
+
+Note: Due to hardware constraints, only the basic version is recommended for Raspberry Pi.
+
+</details>
+
+<details>
+<summary>🐳 <strong>Option 2: Build from Repository</strong></summary>
+
+Build the image locally based on your platform:
+
+```bash
+# Clone the repository
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+
+# For AMD64 (Regular Linux/Windows)
+docker build --platform linux/amd64 \
+  --tag crawl4ai:local \
+  --build-arg INSTALL_TYPE=basic \
+  .
+
+# For ARM64 (M1/M2 Macs, ARM servers)
+docker build --platform linux/arm64 \
+  --tag crawl4ai:local \
+  --build-arg INSTALL_TYPE=basic \
+  .
+```
+
+Build options:
+- INSTALL_TYPE=basic (default): Basic crawling features
+- INSTALL_TYPE=all: Full ML/LLM support
+- ENABLE_GPU=true: Add GPU support
+
+Example with all options:
+```bash
+docker build --platform linux/amd64 \
+  --tag crawl4ai:local \
+  --build-arg INSTALL_TYPE=all \
+  --build-arg ENABLE_GPU=true \
+  .
+```
+
+Run your local build:
+```bash
+# Regular run
+docker run -p 11235:11235 crawl4ai:local
+
+# With increased shared memory
+docker run --shm-size=2gb -p 11235:11235 crawl4ai:local
+```
+
+Test the installation:
+```bash
+curl http://localhost:11235/health
+```
+
+</details>
+
+<details>
+<summary>🐳 <strong>Option 3: Using Docker Compose</strong></summary>
+
+Docker Compose provides a more structured way to run Crawl4AI, especially when dealing with environment variables and multiple configurations.
+
+```bash
+# Clone the repository
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+```
+
+### For AMD64 (Regular Linux/Windows):
+```bash
+# Build and run locally
+docker-compose --profile local-amd64 up
+
+# Run from Docker Hub
+VERSION=basic docker-compose --profile hub-amd64 up   # Basic version
+VERSION=all docker-compose --profile hub-amd64 up     # Full ML/LLM support
+VERSION=gpu docker-compose --profile hub-amd64 up     # GPU support
+```
+
+### For ARM64 (M1/M2 Macs, ARM servers):
+```bash
+# Build and run locally
+docker-compose --profile local-arm64 up
+
+# Run from Docker Hub
+VERSION=basic docker-compose --profile hub-arm64 up   # Basic version
+VERSION=all docker-compose --profile hub-arm64 up     # Full ML/LLM support
+VERSION=gpu docker-compose --profile hub-arm64 up     # GPU support
+```
+
+Environment variables (optional):
+```bash
+# Create a .env file
+CRAWL4AI_API_TOKEN=your_token
+OPENAI_API_KEY=your_openai_key
+CLAUDE_API_KEY=your_claude_key
+```
+
+The compose file includes:
+- Memory management (4GB limit, 1GB reserved)
+- Shared memory volume for browser support
+- Health checks
+- Auto-restart policy
+- All necessary port mappings
+
+Test the installation:
+```bash
+curl http://localhost:11235/health
+```
+
+</details>
+
+<details>
+<summary>🚀 <strong>One-Click Deployment</strong></summary>
+
+Deploy your own instance of Crawl4AI with one click:
+
+[![DigitalOcean Referral Badge](https://web-platforms.sfo2.cdn.digitaloceanspaces.com/WWW/Badge%203.svg)](https://www.digitalocean.com/?repo=https://github.com/unclecode/crawl4ai/tree/0.3.74&refcode=a0780f1bdb3d&utm_campaign=Referral_Invite&utm_medium=Referral_Program&utm_source=badge)
+
+> 💡 **Recommended specs**: 4GB RAM minimum. Select "professional-xs" or higher when deploying for stable operation.
+
+The deploy will:
+- Set up a Docker container with Crawl4AI
+- Configure Playwright and all dependencies
+- Start the FastAPI server on port `11235`
+- Set up health checks and auto-deployment
+
+</details>

docs/examples/amazon_product_extraction_direct_url.py

@@ -0,0 +1,110 @@
+"""
+This example demonstrates how to use JSON CSS extraction to scrape product information 
+from Amazon search results. It shows how to extract structured data like product titles,
+prices, ratings, and other details using CSS selectors.
+"""
+
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+import json
+
+
+async def extract_amazon_products():
+    # Initialize browser config
+    browser_config = BrowserConfig(browser_type="chromium", headless=True)
+
+    # Initialize crawler config with JSON CSS extraction strategy
+    crawler_config = CrawlerRunConfig(
+        extraction_strategy=JsonCssExtractionStrategy(
+            schema={
+                "name": "Amazon Product Search Results",
+                "baseSelector": "[data-component-type='s-search-result']",
+                "fields": [
+                    {
+                        "name": "asin",
+                        "selector": "",
+                        "type": "attribute",
+                        "attribute": "data-asin",
+                    },
+                    {"name": "title", "selector": "h2 a span", "type": "text"},
+                    {
+                        "name": "url",
+                        "selector": "h2 a",
+                        "type": "attribute",
+                        "attribute": "href",
+                    },
+                    {
+                        "name": "image",
+                        "selector": ".s-image",
+                        "type": "attribute",
+                        "attribute": "src",
+                    },
+                    {
+                        "name": "rating",
+                        "selector": ".a-icon-star-small .a-icon-alt",
+                        "type": "text",
+                    },
+                    {
+                        "name": "reviews_count",
+                        "selector": "[data-csa-c-func-deps='aui-da-a-popover'] ~ span span",
+                        "type": "text",
+                    },
+                    {
+                        "name": "price",
+                        "selector": ".a-price .a-offscreen",
+                        "type": "text",
+                    },
+                    {
+                        "name": "original_price",
+                        "selector": ".a-price.a-text-price .a-offscreen",
+                        "type": "text",
+                    },
+                    {
+                        "name": "sponsored",
+                        "selector": ".puis-sponsored-label-text",
+                        "type": "exists",
+                    },
+                    {
+                        "name": "delivery_info",
+                        "selector": "[data-cy='delivery-recipe'] .a-color-base",
+                        "type": "text",
+                        "multiple": True,
+                    },
+                ],
+            }
+        )
+    )
+
+    # Example search URL (you should replace with your actual Amazon URL)
+    url = "https://www.amazon.com/s?k=Samsung+Galaxy+Tab"
+
+    # Use context manager for proper resource handling
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        # Extract the data
+        result = await crawler.arun(url=url, config=crawler_config)
+
+        # Process and print the results
+        if result and result.extracted_content:
+            # Parse the JSON string into a list of products
+            products = json.loads(result.extracted_content)
+
+            # Process each product in the list
+            for product in products:
+                print("\nProduct Details:")
+                print(f"ASIN: {product.get('asin')}")
+                print(f"Title: {product.get('title')}")
+                print(f"Price: {product.get('price')}")
+                print(f"Original Price: {product.get('original_price')}")
+                print(f"Rating: {product.get('rating')}")
+                print(f"Reviews: {product.get('reviews_count')}")
+                print(f"Sponsored: {'Yes' if product.get('sponsored') else 'No'}")
+                if product.get("delivery_info"):
+                    print(f"Delivery: {' '.join(product['delivery_info'])}")
+                print("-" * 80)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(extract_amazon_products())