chore(git): update gitignore patterns

Add new development and tooling related patterns to gitignore:
- Add Next.js build directory (.next/)
- Add various script and documentation files
- Add local development directories (.local, .scripts, .do)
- Add tool-specific files (.codeiumignore, .windsurfrules)

Removes duplicate entries and organizes patterns more clearly.

.claude/settings.local.json

@@ -1,28 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(cd:*)",
-      "Bash(python3:*)",
-      "Bash(python:*)",
-      "Bash(grep:*)",
-      "Bash(mkdir:*)",
-      "Bash(cp:*)",
-      "Bash(rm:*)",
-      "Bash(true)",
-      "Bash(./package-extension.sh:*)",
-      "Bash(find:*)",
-      "Bash(chmod:*)",
-      "Bash(rg:*)",
-      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -A 5 -B 5 \"Script Builder\" docs/md_v2/apps/crawl4ai-assistant/)",
-      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg -A 30 \"generateCode\\(events, format\\)\" docs/md_v2/apps/crawl4ai-assistant/content/content.js)",
-      "Bash(/Users/unclecode/.npm-global/lib/node_modules/@anthropic-ai/claude-code/vendor/ripgrep/arm64-darwin/rg \"<style>\" docs/md_v2/apps/crawl4ai-assistant/index.html -A 5)",
-      "Bash(git checkout:*)",
-      "Bash(docker logs:*)",
-      "Bash(curl:*)",
-      "Bash(docker compose:*)",
-      "Bash(./test-final-integration.sh:*)",
-      "Bash(mv:*)"
-    ]
-  },
-  "enableAllProjectMcpServers": false
-}

.gitattributes

@@ -1,12 +0,0 @@
-# Documentation
-*.html linguist-documentation
-docs/* linguist-documentation
-docs/examples/* linguist-documentation
-docs/md_v2/* linguist-documentation
-
-# Explicitly mark Python as the main language
-*.py linguist-detectable=true
-*.py linguist-language=Python
-
-# Exclude HTML from language statistics
-*.html linguist-detectable=false

.github/DISCUSSION_TEMPLATE/feature-requests.yml

@@ -1,59 +0,0 @@
-title: "[Feature Request]: "
-labels: ["⚙️ New"]
-body:
-  - type: markdown
-    attributes:
-      value: |
-        Thank you for your interest in suggesting a new feature! Before you submit, please take a moment to check if already exists in
-        this discussions category to avoid duplicates. 😊
-
-  - type: textarea
-    id: needs_to_be_done
-    attributes:
-      label: What needs to be done?
-      description: Please describe the feature or functionality you'd like to see.
-      placeholder: "e.g., Return alt text along with images scraped from a webpages in Result"
-    validations:
-      required: true
-
-  - type: textarea
-    id: problem_to_solve
-    attributes:
-      label: What problem does this solve?
-      description: Explain the pain point or issue this feature will help address.
-      placeholder: "e.g., Bypass Captchas added by cloudflare"
-    validations:
-      required: true
-
-  - type: textarea
-    id: target_users
-    attributes:
-      label: Target users/beneficiaries
-      description: Who would benefit from this feature? (e.g., specific teams, developers, users, etc.)
-      placeholder: "e.g., Marketing teams, developers"
-    validations:
-      required: false
-
-  - type: textarea
-    id: current_workarounds
-    attributes:
-      label: Current alternatives/workarounds
-      description: Are there any existing solutions or workarounds? How does this feature improve upon them?
-      placeholder: "e.g., Users manually select the css classes mapped to data fields to extract them"
-    validations:
-      required: false
-
-  - type: markdown
-    attributes:
-      value: |
-        ### 💡 Implementation Ideas
-
-  - type: textarea
-    id: proposed_approach
-    attributes:
-      label: Proposed approach
-      description: Share any ideas you have for how this feature could be implemented. Point out any challenges your foresee
-       and the success metrics for this feature
-      placeholder: "e.g., Implement a breadth first traversal algorithm for scraper"
-    validations:
-      required: false

.github/FUNDING.yml

@@ -1,7 +0,0 @@
-# These are supported funding model platforms
-
-# GitHub Sponsors
-github: unclecode
-
-# Custom links for enterprise inquiries (uncomment when ready)
-# custom: ["https://crawl4ai.com/enterprise"]

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,127 +0,0 @@
-name: Bug Report
-description: Report a bug with the Crawl4AI.
-title: "[Bug]: "
-labels: ["🐞 Bug","🩺 Needs Triage"]
-body:
-  - type: input
-    id: crawl4ai_version
-    attributes:
-      label: crawl4ai version
-      description: Specify the version of crawl4ai you are using.
-      placeholder: "e.g., 2.0.0"
-    validations:
-      required: true
-
-  - type: textarea
-    id: expected_behavior
-    attributes:
-      label: Expected Behavior
-      description: Describe what you expected to happen.
-      placeholder: "Provide a detailed explanation of the expected outcome."
-    validations:
-      required: true
-
-  - type: textarea
-    id: current_behavior
-    attributes:
-      label: Current Behavior
-      description: Describe what is happening instead of the expected behavior.
-      placeholder: "Describe the actual result or issue you encountered."
-    validations:
-      required: true
-
-  - type: dropdown
-    id: reproducible
-    attributes:
-      label: Is this reproducible?
-      description: Indicate whether this bug can be reproduced consistently.
-      options:
-        - "Yes"
-        - "No"
-    validations:
-      required: true
-
-  - type: textarea
-    id: inputs
-    attributes:
-      label: Inputs Causing the Bug
-      description: Provide details about the inputs causing the issue.
-      placeholder: |
-        - URL(s): 
-        - Settings used: 
-        - Input data (if applicable): 
-      render: bash
-  
-  - type: textarea
-    id: steps_to_reproduce
-    attributes:
-      label: Steps to Reproduce
-      description: Provide step-by-step instructions to reproduce the issue.
-      placeholder: |
-        1. Go to...
-        2. Click on...
-        3. Observe the issue...
-      render: bash
-  
-  - type: textarea
-    id: code_snippets
-    attributes:
-      label: Code snippets
-      description: Provide code snippets(if any). Add comments as necessary
-      placeholder: print("Hello world")
-      render: python
-
-  # Header Section with Title
-  - type: markdown
-    attributes:
-      value: |
-          ## Supporting Information
-          Please provide the following details to help us understand and resolve your issue. This will assist us in reproducing and diagnosing the problem
-
-  - type: input
-    id: os
-    attributes:
-      label: OS
-      description: Please provide the operating system & distro where the issue occurs.
-      placeholder: "e.g., Windows, macOS, Linux"
-    validations:
-      required: true
-
-  - type: input
-    id: python_version
-    attributes:
-      label: Python version
-      description: Specify the Python version being used.
-      placeholder: "e.g., 3.8.5"
-    validations:
-      required: true
-
-  # Browser Field
-  - type: input
-    id: browser
-    attributes:
-      label: Browser
-      description: Provide the name of the browser you are using.
-      placeholder: "e.g., Chrome, Firefox, Safari"
-    validations:
-      required: false
-
-  # Browser Version Field
-  - type: input
-    id: browser_version
-    attributes:
-      label: Browser version
-      description: Provide the version of the browser you are using.
-      placeholder: "e.g., 91.0.4472.124"
-    validations:
-      required: false
-
-  # Error Logs Field (Text Area)
-  - type: textarea
-    id: error_logs
-    attributes:
-      label: Error logs & Screenshots (if applicable)
-      description: If you encountered any errors, please provide the error logs. Attach any relevant screenshots to help us understand the issue.
-      placeholder: "Paste error logs here and attach your screenshots"
-    validations:
-      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Feature Requests
-    url: https://github.com/unclecode/crawl4ai/discussions/categories/feature-requests
-    about: "Suggest new features or enhancements for Crawl4AI"
-  - name: Forums - Q&A
-    url: https://github.com/unclecode/crawl4ai/discussions/categories/forums-q-a
-    about: "Ask questions or engage in general discussions about Crawl4AI"

.github/pull_request_template.md

@@ -1,19 +0,0 @@
-## Summary
-Please include a summary of the change and/or which issues are fixed.
-
-eg: `Fixes #123` (Tag GitHub issue numbers in this format, so it automatically links the issues with your PR)
-
-## List of files changed and why
-eg: quickstart.py - To update the example as per new changes
-
-## How Has This Been Tested?
-Please describe the tests that you ran to verify your changes.
-
-## Checklist:
-
-- [ ] My code follows the style guidelines of this project
-- [ ] I have performed a self-review of my own code
-- [ ] I have commented my code, particularly in hard-to-understand areas
-- [ ] I have made corresponding changes to the documentation
-- [ ] I have added/updated unit tests that prove my fix is effective or that my feature works
-- [ ] New and existing unit tests pass locally with my changes

.github/workflows/main.yml

@@ -1,46 +0,0 @@
-name: Discord GitHub Notifications
-
-on:
-  issues:
-    types: [opened]
-  issue_comment:
-    types: [created]
-  pull_request:
-    types: [opened]
-  discussion:
-    types: [created]
-  watch:
-    types: [started]
-
-jobs:
-  notify-discord:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Send to Google Apps Script (Stars only)
-        if: github.event_name == 'watch'
-        run: |
-          curl -fSs -X POST "${{ secrets.GOOGLE_SCRIPT_ENDPOINT }}" \
-            -H 'Content-Type: application/json' \
-            -d '{"url":"${{ github.event.sender.html_url }}"}'
-      - name: Set webhook based on event type
-        id: set-webhook
-        run: |
-          if [ "${{ github.event_name }}" == "discussion" ]; then
-            echo "webhook=${{ secrets.DISCORD_DISCUSSIONS_WEBHOOK }}" >> $GITHUB_OUTPUT
-          elif [ "${{ github.event_name }}" == "watch" ]; then
-            echo "webhook=${{ secrets.DISCORD_STAR_GAZERS }}" >> $GITHUB_OUTPUT
-          else
-            echo "webhook=${{ secrets.DISCORD_WEBHOOK }}" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Discord Notification
-        uses: Ilshidur/action-discord@master
-        env:
-          DISCORD_WEBHOOK: ${{ steps.set-webhook.outputs.webhook }}
-        with:
-          args: |
-            ${{ github.event_name == 'issues' && format('📣 New issue created: **{0}** by {1} - {2}', github.event.issue.title, github.event.issue.user.login, github.event.issue.html_url) || 
-            github.event_name == 'issue_comment' && format('💬 New comment on issue **{0}** by {1} - {2}', github.event.issue.title, github.event.comment.user.login, github.event.comment.html_url) ||
-            github.event_name == 'pull_request' && format('🔄 New PR opened: **{0}** by {1} - {2}', github.event.pull_request.title, github.event.pull_request.user.login, github.event.pull_request.html_url) ||
-            github.event_name == 'watch' && format('⭐ {0} starred Crawl4AI 🥳! Check out their profile: {1}', github.event.sender.login, github.event.sender.html_url) ||
-            format('💬 New discussion started: **{0}** by {1} - {2}', github.event.discussion.title, github.event.discussion.user.login, github.event.discussion.html_url) }}

.github/workflows/release.yml

@@ -1,142 +0,0 @@
-name: Release Pipeline
-on:
-  push:
-    tags:
-      - 'v*'
-      - '!test-v*'  # Exclude test tags
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write  # Required for creating releases
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-      
-      - name: Extract version from tag
-        id: get_version
-        run: |
-          TAG_VERSION=${GITHUB_REF#refs/tags/v}
-          echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
-          echo "Releasing version: $TAG_VERSION"
-      
-      - name: Install package dependencies
-        run: |
-          pip install -e .
-      
-      - name: Check version consistency
-        run: |
-          TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
-          PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
-          
-          echo "Tag version: $TAG_VERSION"
-          echo "Package version: $PACKAGE_VERSION"
-          
-          if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
-            echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
-            echo "Please update crawl4ai/__version__.py to match the tag version"
-            exit 1
-          fi
-          echo "✅ Version check passed: $TAG_VERSION"
-      
-      - name: Install build dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install build twine
-      
-      - name: Build package
-        run: python -m build
-      
-      - name: Check package
-        run: twine check dist/*
-      
-      - name: Upload to PyPI
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        run: |
-          echo "📦 Uploading to PyPI..."
-          twine upload dist/*
-          echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/"
-      
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      
-      - name: Extract major and minor versions
-        id: versions
-        run: |
-          VERSION=${{ steps.get_version.outputs.VERSION }}
-          MAJOR=$(echo $VERSION | cut -d. -f1)
-          MINOR=$(echo $VERSION | cut -d. -f1-2)
-          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
-          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
-      
-      - name: Build and push Docker images
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
-            unclecode/crawl4ai:latest
-          platforms: linux/amd64,linux/arm64
-      
-      - name: Create GitHub Release
-        uses: softprops/action-gh-release@v2
-        with:
-          tag_name: v${{ steps.get_version.outputs.VERSION }}
-          name: Release v${{ steps.get_version.outputs.VERSION }}
-          body: |
-            ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released!
-            
-            ### 📦 Installation
-            
-            **PyPI:**
-            ```bash
-            pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}
-            ```
-            
-            **Docker:**
-            ```bash
-            docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
-            docker pull unclecode/crawl4ai:latest
-            ```
-            
-            ### 📝 What's Changed
-            See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
-          draft: false
-          prerelease: false
-          token: ${{ secrets.GITHUB_TOKEN }}
-      
-      - name: Summary
-        run: |
-          echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 📦 PyPI Package" >> $GITHUB_STEP_SUMMARY
-          echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
-          echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
-          echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
-          echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/test-release.yml.disabled

@@ -1,116 +0,0 @@
-name: Test Release Pipeline
-on:
-  push:
-    tags:
-      - 'test-v*'
-
-jobs:
-  test-release:
-    runs-on: ubuntu-latest
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-      
-      - name: Extract version from tag
-        id: get_version
-        run: |
-          TAG_VERSION=${GITHUB_REF#refs/tags/test-v}
-          echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
-          echo "Testing with version: $TAG_VERSION"
-      
-      - name: Install package dependencies
-        run: |
-          pip install -e .
-      
-      - name: Check version consistency
-        run: |
-          TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
-          PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
-          
-          echo "Tag version: $TAG_VERSION"
-          echo "Package version: $PACKAGE_VERSION"
-          
-          if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
-            echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
-            echo "Please update crawl4ai/__version__.py to match the tag version"
-            exit 1
-          fi
-          echo "✅ Version check passed: $TAG_VERSION"
-      
-      - name: Install build dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install build twine
-      
-      - name: Build package
-        run: python -m build
-      
-      - name: Check package
-        run: twine check dist/*
-      
-      - name: Upload to Test PyPI
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }}
-        run: |
-          echo "📦 Uploading to Test PyPI..."
-          twine upload --repository testpypi dist/* || {
-            if [ $? -eq 1 ]; then
-              echo "⚠️ Upload failed - likely version already exists on Test PyPI"
-              echo "Continuing anyway for test purposes..."
-            else
-              exit 1
-            fi
-          }
-          echo "✅ Test PyPI step complete"
-      
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      
-      - name: Build and push Docker test images
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }}
-            unclecode/crawl4ai:test-latest
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-      
-      - name: Summary
-        run: |
-          echo "## 🎉 Test Release Complete!" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 📦 Test PyPI Package" >> $GITHUB_STEP_SUMMARY
-          echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
-          echo "- URL: https://test.pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
-          echo "- Install: \`pip install -i https://test.pypi.org/simple/ crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 🐳 Docker Test Images" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:test-latest\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 🧹 Cleanup Commands" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
-          echo "# Remove test tag" >> $GITHUB_STEP_SUMMARY
-          echo "git tag -d test-v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
-          echo "git push origin :test-v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "# Remove Docker test images" >> $GITHUB_STEP_SUMMARY
-          echo "docker rmi unclecode/crawl4ai:test-${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
-          echo "docker rmi unclecode/crawl4ai:test-latest" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -1,6 +1,3 @@
-# Scripts folder (private tools)
-.scripts/
-
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -229,50 +226,8 @@ tree.md
 .local
 .do
 /plans
-plans/
-
-# Codeium
 .codeiumignore
 todo/
 
-# Continue development files
-.continue/
-.continuerc.json
-continue.lock
-continue_core.log
-contextProviders/
-continue_workspace/
-.continue-cache/
-continue_config.json
-
-# Continue temporary files
-.continue-temp/
-.continue-logs/
-.continue-downloads/
-
-# Continue VS Code specific
-.vscode-continue/
-.vscode-continue-cache/
-
-.prompts/
-
-.llm.env
-.private/
-
-CLAUDE_MONITOR.md
-CLAUDE.md
-.claude/
-
-scripts/
-
-tests/**/test_site
-tests/**/reports
-tests/**/benchmark_reports
-
-docs/**/data
-.codecat/
-
-docs/apps/linkdin/debug*/
-docs/apps/linkdin/samples/insights/*
-docs/md_v2/marketplace/backend/uploads/
-docs/md_v2/marketplace/backend/marketplace.db
+# windsurf rules
+.windsurfrules

CODE_OF_CONDUCT.md

@@ -1,131 +0,0 @@
-# Crawl4AI Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email address,
-  without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official email address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-unclecode@crawl4ai.com. All complaints will be reviewed and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.1, available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
-
-Community Impact Guidelines were inspired by
-[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
-[https://www.contributor-covenant.org/translations][translations].
-
-[homepage]: https://www.contributor-covenant.org
-[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
-[Mozilla CoC]: https://github.com/mozilla/diversity
-[FAQ]: https://www.contributor-covenant.org/faq
-[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTORS.md

@@ -6,44 +6,21 @@ We would like to thank the following people for their contributions to Crawl4AI:
 
 - [Unclecode](https://github.com/unclecode) - Project Creator and Main Developer
 - [Nasrin](https://github.com/ntohidi) - Project Manager and Developer
-- [Aravind Karnam](https://github.com/aravindkarnam) - Head of Community and Product 
+- [Aravind Karnam](https://github.com/aravindkarnam) - Developer
 
 ## Community Contributors
 
-- [aadityakanjolia4](https://github.com/aadityakanjolia4) - Fix for `CustomHTML2Text` is not defined.
 - [FractalMind](https://github.com/FractalMind) - Created the first official Docker Hub image and fixed Dockerfile errors
 - [ketonkss4](https://github.com/ketonkss4) - Identified Selenium's new capabilities, helping reduce dependencies
 - [jonymusky](https://github.com/jonymusky) - Javascript execution documentation, and wait_for
 - [datehoer](https://github.com/datehoer) - Add browser prxy support
 
-## Pull Requests
-
-- [dvschuyl](https://github.com/dvschuyl) - AsyncPlaywrightCrawlerStrategy page-evaluate context destroyed by navigation [#304](https://github.com/unclecode/crawl4ai/pull/304)
-- [nelzomal](https://github.com/nelzomal) - Enhance development installation instructions [#286](https://github.com/unclecode/crawl4ai/pull/286)
-- [HamzaFarhan](https://github.com/HamzaFarhan) - Handled the cases where markdown_with_citations, references_markdown, and filtered_html might not be defined [#293](https://github.com/unclecode/crawl4ai/pull/293)
-- [NanmiCoder](https://github.com/NanmiCoder) - fix: crawler strategy exception handling and fixes [#271](https://github.com/unclecode/crawl4ai/pull/271)
-- [paulokuong](https://github.com/paulokuong) - fix: RAWL4_AI_BASE_DIRECTORY should be Path object instead of string [#298](https://github.com/unclecode/crawl4ai/pull/298)
-
-#### Feb-Alpha-1
-- [sufianuddin](https://github.com/sufianuddin) - fix: [Documentation for JsonCssExtractionStrategy](https://github.com/unclecode/crawl4ai/issues/651)
-- [tautikAg](https://github.com/tautikAg) - fix: [Markdown output has incorect spacing](https://github.com/unclecode/crawl4ai/issues/599)
-- [cardit1](https://github.com/cardit1) - fix: ['AsyncPlaywrightCrawlerStrategy' object has no attribute 'downloads_path'](https://github.com/unclecode/crawl4ai/issues/585)
-- [dmurat](https://github.com/dmurat) - fix: [ Incorrect rendering of inline code inside of links ](https://github.com/unclecode/crawl4ai/issues/583)
-- [Sparshsing](https://github.com/Sparshsing) - fix: [Relative Urls in the webpage not extracted properly ](https://github.com/unclecode/crawl4ai/issues/570)
-
-
-
 ## Other Contributors
 
 - [Gokhan](https://github.com/gkhngyk) 
 - [Shiv Kumar](https://github.com/shivkumar0757)
 - [QIN2DIM](https://github.com/QIN2DIM)
 
-#### Typo fixes
-- [ssoydan](https://github.com/ssoydan)
-- [Darshan](https://github.com/Darshan2104)
-- [tuhinmallick](https://github.com/tuhinmallick)
-
 ## Acknowledgements
 
 We also want to thank all the users who have reported bugs, suggested features, or helped in any other way to make Crawl4AI better.

Dockerfile

@@ -1,200 +0,0 @@
-FROM python:3.12-slim-bookworm AS build
-
-# C4ai version
-ARG C4AI_VER=0.7.0-r1
-ENV C4AI_VERSION=$C4AI_VER
-LABEL c4ai.version=$C4AI_VER
-
-# Set build arguments
-ARG APP_HOME=/app
-ARG GITHUB_REPO=https://github.com/unclecode/crawl4ai.git
-ARG GITHUB_BRANCH=main
-ARG USE_LOCAL=true
-
-ENV PYTHONFAULTHANDLER=1 \
-    PYTHONHASHSEED=random \
-    PYTHONUNBUFFERED=1 \
-    PIP_NO_CACHE_DIR=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_DEFAULT_TIMEOUT=100 \
-    DEBIAN_FRONTEND=noninteractive \
-    REDIS_HOST=localhost \
-    REDIS_PORT=6379
-
-ARG PYTHON_VERSION=3.12
-ARG INSTALL_TYPE=default
-ARG ENABLE_GPU=false
-ARG TARGETARCH
-
-LABEL maintainer="unclecode"
-LABEL description="🔥🕷️ Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
-LABEL version="1.0"
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    build-essential \
-    curl \
-    wget \
-    gnupg \
-    git \
-    cmake \
-    pkg-config \
-    python3-dev \
-    libjpeg-dev \
-    redis-server \
-    supervisor \
-    && apt-get clean \ 
-    && rm -rf /var/lib/apt/lists/*
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libglib2.0-0 \
-    libnss3 \
-    libnspr4 \
-    libatk1.0-0 \
-    libatk-bridge2.0-0 \
-    libcups2 \
-    libdrm2 \
-    libdbus-1-3 \
-    libxcb1 \
-    libxkbcommon0 \
-    libx11-6 \
-    libxcomposite1 \
-    libxdamage1 \
-    libxext6 \
-    libxfixes3 \
-    libxrandr2 \
-    libgbm1 \
-    libpango-1.0-0 \
-    libcairo2 \
-    libasound2 \
-    libatspi2.0-0 \
-    && apt-get clean \ 
-    && rm -rf /var/lib/apt/lists/*
-
-RUN apt-get update && apt-get dist-upgrade -y \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ] ; then \
-    apt-get update && apt-get install -y --no-install-recommends \
-    nvidia-cuda-toolkit \
-    && apt-get clean \ 
-    && rm -rf /var/lib/apt/lists/* ; \
-else \
-    echo "Skipping NVIDIA CUDA Toolkit installation (unsupported platform or GPU disabled)"; \
-fi
-
-RUN if [ "$TARGETARCH" = "arm64" ]; then \
-    echo "🦾 Installing ARM-specific optimizations"; \
-    apt-get update && apt-get install -y --no-install-recommends \
-    libopenblas-dev \
-    && apt-get clean \ 
-    && rm -rf /var/lib/apt/lists/*; \
-elif [ "$TARGETARCH" = "amd64" ]; then \
-    echo "🖥️ Installing AMD64-specific optimizations"; \
-    apt-get update && apt-get install -y --no-install-recommends \
-    libomp-dev \
-    && apt-get clean \ 
-    && rm -rf /var/lib/apt/lists/*; \
-else \
-    echo "Skipping platform-specific optimizations (unsupported platform)"; \
-fi
-
-# Create a non-root user and group
-RUN groupadd -r appuser && useradd --no-log-init -r -g appuser appuser
-
-# Create and set permissions for appuser home directory
-RUN mkdir -p /home/appuser && chown -R appuser:appuser /home/appuser
-
-WORKDIR ${APP_HOME}
-
-RUN echo '#!/bin/bash\n\
-if [ "$USE_LOCAL" = "true" ]; then\n\
-    echo "📦 Installing from local source..."\n\
-    pip install --no-cache-dir /tmp/project/\n\
-else\n\
-    echo "🌐 Installing from GitHub..."\n\
-    for i in {1..3}; do \n\
-        git clone --branch ${GITHUB_BRANCH} ${GITHUB_REPO} /tmp/crawl4ai && break || \n\
-        { echo "Attempt $i/3 failed! Taking a short break... ☕"; sleep 5; }; \n\
-    done\n\
-    pip install --no-cache-dir /tmp/crawl4ai\n\
-fi' > /tmp/install.sh && chmod +x /tmp/install.sh
-
-COPY . /tmp/project/
-
-# Copy supervisor config first (might need root later, but okay for now)
-COPY deploy/docker/supervisord.conf .
-
-COPY deploy/docker/requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
-        pip install --no-cache-dir \
-            torch \
-            torchvision \
-            torchaudio \
-            scikit-learn \
-            nltk \
-            transformers \
-            tokenizers && \
-        python -m nltk.downloader punkt stopwords ; \
-    fi
-
-RUN if [ "$INSTALL_TYPE" = "all" ] ; then \
-        pip install "/tmp/project/[all]" && \
-        python -m crawl4ai.model_loader ; \
-    elif [ "$INSTALL_TYPE" = "torch" ] ; then \
-        pip install "/tmp/project/[torch]" ; \
-    elif [ "$INSTALL_TYPE" = "transformer" ] ; then \
-        pip install "/tmp/project/[transformer]" && \
-        python -m crawl4ai.model_loader ; \
-    else \
-        pip install "/tmp/project" ; \
-    fi
-
-RUN pip install --no-cache-dir --upgrade pip && \
-    /tmp/install.sh && \
-    python -c "import crawl4ai; print('✅ crawl4ai is ready to rock!')" && \
-    python -c "from playwright.sync_api import sync_playwright; print('✅ Playwright is feeling dramatic!')"
-
-RUN crawl4ai-setup
-
-RUN playwright install --with-deps
-
-RUN mkdir -p /home/appuser/.cache/ms-playwright \
-    && cp -r /root/.cache/ms-playwright/chromium-* /home/appuser/.cache/ms-playwright/ \
-    && chown -R appuser:appuser /home/appuser/.cache/ms-playwright
-
-RUN crawl4ai-doctor
-
-# Copy application code
-COPY deploy/docker/* ${APP_HOME}/
-
-# copy the playground + any future static assets
-COPY deploy/docker/static ${APP_HOME}/static
-
-# Change ownership of the application directory to the non-root user
-RUN chown -R appuser:appuser ${APP_HOME}
-
-# give permissions to redis persistence dirs if used
-RUN mkdir -p /var/lib/redis /var/log/redis && chown -R appuser:appuser /var/lib/redis /var/log/redis
-
-HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
-    CMD bash -c '\
-    MEM=$(free -m | awk "/^Mem:/{print \$2}"); \
-    if [ $MEM -lt 2048 ]; then \
-        echo "⚠️ Warning: Less than 2GB RAM available! Your container might need a memory boost! 🚀"; \
-        exit 1; \
-    fi && \
-    redis-cli ping > /dev/null && \
-    curl -f http://localhost:11235/health || exit 1'
-
-EXPOSE 6379
-# Switch to the non-root user before starting the application
-USER appuser
-
-# Set environment variables to ptoduction
-ENV PYTHON_ENV=production 
-
-# Start the application using supervisord
-CMD ["supervisord", "-c", "supervisord.conf"]

JOURNAL.md

@@ -1,339 +0,0 @@
-# Development Journal
-
-This journal tracks significant feature additions, bug fixes, and architectural decisions in the crawl4ai project. It serves as both documentation and a historical record of the project's evolution.
-
-## [2025-04-17] Added Content Source Selection for Markdown Generation
-
-**Feature:** Configurable content source for markdown generation
-
-**Changes Made:**
-1. Added `content_source: str = "cleaned_html"` parameter to `MarkdownGenerationStrategy` class
-2. Updated `DefaultMarkdownGenerator` to accept and pass the content source parameter
-3. Renamed the `cleaned_html` parameter to `input_html` in the `generate_markdown` method
-4. Modified `AsyncWebCrawler.aprocess_html` to select the appropriate HTML source based on the generator's config
-5. Added `preprocess_html_for_schema` import in `async_webcrawler.py`
-
-**Implementation Details:**
-- Added a new `content_source` parameter to specify which HTML input to use for markdown generation
-- Options include: "cleaned_html" (default), "raw_html", and "fit_html"
-- Used a dictionary dispatch pattern in `aprocess_html` to select the appropriate HTML source
-- Added proper error handling with fallback to cleaned_html if content source selection fails
-- Ensured backward compatibility by defaulting to "cleaned_html" option
-
-**Files Modified:**
-- `crawl4ai/markdown_generation_strategy.py`: Added content_source parameter and updated the method signature
-- `crawl4ai/async_webcrawler.py`: Added HTML source selection logic and updated imports
-
-**Examples:**
-- Created `docs/examples/content_source_example.py` demonstrating how to use the new parameter
-
-**Challenges:**
-- Maintaining backward compatibility while reorganizing the parameter flow
-- Ensuring proper error handling for all content source options
-- Making the change with minimal code modifications
-
-**Why This Feature:**
-The content source selection feature allows users to choose which HTML content to use as input for markdown generation:
-1. "cleaned_html" - Uses the post-processed HTML after scraping strategy (original behavior)
-2. "raw_html" - Uses the original raw HTML directly from the web page
-3. "fit_html" - Uses the preprocessed HTML optimized for schema extraction
-
-This feature provides greater flexibility in how users generate markdown, enabling them to:
-- Capture more detailed content from the original HTML when needed
-- Use schema-optimized HTML when working with structured data
-- Choose the approach that best suits their specific use case
-## [2025-04-17] Implemented High Volume Stress Testing Solution for SDK
-
-**Feature:** Comprehensive stress testing framework using `arun_many` and the dispatcher system to evaluate performance, concurrency handling, and identify potential issues under high-volume crawling scenarios.
-
-**Changes Made:**
-1.  Created a dedicated stress testing framework in the `benchmarking/` (or similar) directory.
-2.  Implemented local test site generation (`SiteGenerator`) with configurable heavy HTML pages.
-3.  Added basic memory usage tracking (`SimpleMemoryTracker`) using platform-specific commands (avoiding `psutil` dependency for this specific test).
-4.  Utilized `CrawlerMonitor` from `crawl4ai` for rich terminal UI and real-time monitoring of test progress and dispatcher activity.
-5.  Implemented detailed result summary saving (JSON) and memory sample logging (CSV).
-6.  Developed `run_benchmark.py` to orchestrate tests with predefined configurations.
-7.  Created `run_all.sh` as a simple wrapper for `run_benchmark.py`.
-
-**Implementation Details:**
--   Generates a local test site with configurable pages containing heavy text and image content.
--   Uses Python's built-in `http.server` for local serving, minimizing network variance.
--   Leverages `crawl4ai`'s `arun_many` method for processing URLs.
--   Utilizes `MemoryAdaptiveDispatcher` to manage concurrency via the `max_sessions` parameter (note: memory adaptation features require `psutil`, not used by `SimpleMemoryTracker`).
--   Tracks memory usage via `SimpleMemoryTracker`, recording samples throughout test execution to a CSV file.
--   Uses `CrawlerMonitor` (which uses the `rich` library) for clear terminal visualization and progress reporting directly from the dispatcher.
--   Stores detailed final metrics in a JSON summary file.
-
-**Files Created/Updated:**
--   `stress_test_sdk.py`: Main stress testing implementation using `arun_many`.
--   `benchmark_report.py`: (Assumed) Report generator for comparing test results.
--   `run_benchmark.py`: Test runner script with predefined configurations.
--   `run_all.sh`: Simple bash script wrapper for `run_benchmark.py`.
--   `USAGE.md`: Comprehensive documentation on usage and interpretation (updated).
-
-**Testing Approach:**
--   Creates a controlled, reproducible test environment with a local HTTP server.
--   Processes URLs using `arun_many`, allowing the dispatcher to manage concurrency up to `max_sessions`.
--   Optionally logs per-batch summaries (when not in streaming mode) after processing chunks.
--   Supports different test sizes via `run_benchmark.py` configurations.
--   Records memory samples via platform commands for basic trend analysis.
--   Includes cleanup functionality for the test environment.
-
-**Challenges:**
--   Ensuring proper cleanup of HTTP server processes.
--   Getting reliable memory tracking across platforms without adding heavy dependencies (`psutil`) to this specific test script.
--   Designing `run_benchmark.py` to correctly pass arguments to `stress_test_sdk.py`.
-
-**Why This Feature:**
-The high volume stress testing solution addresses critical needs for ensuring Crawl4AI's `arun_many` reliability:
-1.  Provides a reproducible way to evaluate performance under concurrent load.
-2.  Allows testing the dispatcher's concurrency control (`max_session_permit`) and queue management.
-3.  Enables performance tuning by observing throughput (`URLs/sec`) under different `max_sessions` settings.
-4.  Creates a controlled environment for testing `arun_many` behavior.
-5.  Supports continuous integration by providing deterministic test conditions for `arun_many`.
-
-**Design Decisions:**
--   Chose local site generation for reproducibility and isolation from network issues.
--   Utilized the built-in `CrawlerMonitor` for real-time feedback, leveraging its `rich` integration.
--   Implemented optional per-batch logging in `stress_test_sdk.py` (when not streaming) to provide chunk-level summaries alongside the continuous monitor.
--   Adopted `arun_many` with a `MemoryAdaptiveDispatcher` as the core mechanism for parallel execution, reflecting the intended SDK usage.
--   Created `run_benchmark.py` to simplify running standard test configurations.
--   Used `SimpleMemoryTracker` to provide basic memory insights without requiring `psutil` for this particular test runner.
-
-**Future Enhancements to Consider:**
--   Create a separate test variant that *does* use `psutil` to specifically stress the memory-adaptive features of the dispatcher.
--   Add support for generated JavaScript content.
--   Add support for Docker-based testing with explicit memory limits.
--   Enhance `benchmark_report.py` to provide more sophisticated analysis of performance and memory trends from the generated JSON/CSV files.
-
----
-
-## [2025-04-17] Refined Stress Testing System Parameters and Execution
-
-**Changes Made:**
-1.  Corrected `run_benchmark.py` and `stress_test_sdk.py` to use `--max-sessions` instead of the incorrect `--workers` parameter, accurately reflecting dispatcher configuration.
-2.  Updated `run_benchmark.py` argument handling to correctly pass all relevant custom parameters (including `--stream`, `--monitor-mode`, etc.) to `stress_test_sdk.py`.
-3.  (Assuming changes in `benchmark_report.py`) Applied dark theme to benchmark reports for better readability.
-4.  (Assuming changes in `benchmark_report.py`) Improved visualization code to eliminate matplotlib warnings.
-5.  Updated `run_benchmark.py` to provide clickable `file://` links to generated reports in the terminal output.
-6.  Updated `USAGE.md` with comprehensive parameter descriptions reflecting the final script arguments.
-7.  Updated `run_all.sh` wrapper to correctly invoke `run_benchmark.py` with flexible arguments.
-
-**Details of Changes:**
-
-1.  **Parameter Correction (`--max-sessions`)**:
-    *   Identified the fundamental misunderstanding where `--workers` was used incorrectly.
-    *   Refactored `stress_test_sdk.py` to accept `--max-sessions` and configure the `MemoryAdaptiveDispatcher`'s `max_session_permit` accordingly.
-    *   Updated `run_benchmark.py` argument parsing and command construction to use `--max-sessions`.
-    *   Updated `TEST_CONFIGS` in `run_benchmark.py` to use `max_sessions`.
-
-2.  **Argument Handling (`run_benchmark.py`)**:
-    *   Improved logic to collect all command-line arguments provided to `run_benchmark.py`.
-    *   Ensured all relevant arguments (like `--stream`, `--monitor-mode`, `--port`, `--use-rate-limiter`, etc.) are correctly forwarded when calling `stress_test_sdk.py` as a subprocess.
-
-3.  **Dark Theme & Visualization Fixes (Assumed in `benchmark_report.py`)**:
-    *   (Describes changes assumed to be made in the separate reporting script).
-
-4.  **Clickable Links (`run_benchmark.py`)**:
-    *   Added logic to find the latest HTML report and PNG chart in the `benchmark_reports` directory after `benchmark_report.py` runs.
-    *   Used `pathlib` to generate correct `file://` URLs for terminal output.
-
-5.  **Documentation Improvements (`USAGE.md`)**:
-    *   Rewrote sections to explain `arun_many`, dispatchers, and `--max-sessions`.
-    *   Updated parameter tables for all scripts (`stress_test_sdk.py`, `run_benchmark.py`).
-    *   Clarified the difference between batch and streaming modes and their effect on logging.
-    *   Updated examples to use correct arguments.
-
-**Files Modified:**
--   `stress_test_sdk.py`: Changed `--workers` to `--max-sessions`, added new arguments, used `arun_many`.
--   `run_benchmark.py`: Changed argument handling, updated configs, calls `stress_test_sdk.py`.
--   `run_all.sh`: Updated to call `run_benchmark.py` correctly.
--   `USAGE.md`: Updated documentation extensively.
--   `benchmark_report.py`: (Assumed modifications for dark theme and viz fixes).
-
-**Testing:**
--   Verified that `--max-sessions` correctly limits concurrency via the `CrawlerMonitor` output.
--   Confirmed that custom arguments passed to `run_benchmark.py` are forwarded to `stress_test_sdk.py`.
--   Validated clickable links work in supporting terminals.
--   Ensured documentation matches the final script parameters and behavior.
-
-**Why These Changes:**
-These refinements correct the fundamental approach of the stress test to align with `crawl4ai`'s actual architecture and intended usage:
-1.  Ensures the test evaluates the correct components (`arun_many`, `MemoryAdaptiveDispatcher`).
-2.  Makes test configurations more accurate and flexible.
-3.  Improves the usability of the testing framework through better argument handling and documentation.
-
-
-**Future Enhancements to Consider:**
-- Add support for generated JavaScript content to test JS rendering performance
-- Implement more sophisticated memory analysis like generational garbage collection tracking
-- Add support for Docker-based testing with memory limits to force OOM conditions
-- Create visualization tools for analyzing memory usage patterns across test runs
-- Add benchmark comparisons between different crawler versions or configurations
-
-## [2025-04-17] Fixed Issues in Stress Testing System
-
-**Changes Made:**
-1. Fixed custom parameter handling in run_benchmark.py
-2. Applied dark theme to benchmark reports for better readability
-3. Improved visualization code to eliminate matplotlib warnings
-4. Added clickable links to generated reports in terminal output
-5. Enhanced documentation with comprehensive parameter descriptions
-
-**Details of Changes:**
-
-1. **Custom Parameter Handling Fix**
-   - Identified bug where custom URL count was being ignored in run_benchmark.py
-   - Rewrote argument handling to use a custom args dictionary
-   - Properly passed parameters to the test_simple_stress.py command
-   - Added better UI indication of custom parameters in use
-
-2. **Dark Theme Implementation**
-   - Added complete dark theme to HTML benchmark reports
-   - Applied dark styling to all visualization components
-   - Used Nord-inspired color palette for charts and graphs
-   - Improved contrast and readability for data visualization
-   - Updated text colors and backgrounds for better eye comfort
-
-3. **Matplotlib Warning Fixes**
-   - Resolved warnings related to improper use of set_xticklabels()
-   - Implemented correct x-axis positioning for bar charts
-   - Ensured proper alignment of bar labels and data points
-   - Updated plotting code to use modern matplotlib practices
-
-4. **Documentation Improvements**
-   - Created comprehensive USAGE.md with detailed instructions
-   - Added parameter documentation for all scripts
-   - Included examples for all common use cases
-   - Provided detailed explanations for interpreting results
-   - Added troubleshooting guide for common issues
-
-**Files Modified:**
-- `tests/memory/run_benchmark.py`: Fixed custom parameter handling
-- `tests/memory/benchmark_report.py`: Added dark theme and fixed visualization warnings
-- `tests/memory/run_all.sh`: Added clickable links to reports
-- `tests/memory/USAGE.md`: Created comprehensive documentation
-
-**Testing:**
-- Verified that custom URL counts are now correctly used
-- Confirmed dark theme is properly applied to all report elements
-- Checked that matplotlib warnings are no longer appearing
-- Validated clickable links to reports work in terminals that support them
-
-**Why These Changes:**
-These improvements address several usability issues with the stress testing system:
-1. Better parameter handling ensures test configurations work as expected
-2. Dark theme reduces eye strain during extended test review sessions
-3. Fixing visualization warnings improves code quality and output clarity
-4. Enhanced documentation makes the system more accessible for future use
-
-**Future Enhancements:**
-- Add additional visualization options for different types of analysis
-- Implement theme toggle to support both light and dark preferences
-- Add export options for embedding reports in other documentation
-- Create dedicated CI/CD integration templates for automated testing
-
-## [2025-04-09] Added MHTML Capture Feature
-
-**Feature:** MHTML snapshot capture of crawled pages
-
-**Changes Made:**
-1. Added `capture_mhtml: bool = False` parameter to `CrawlerRunConfig` class
-2. Added `mhtml: Optional[str] = None` field to `CrawlResult` model
-3. Added `mhtml_data: Optional[str] = None` field to `AsyncCrawlResponse` class
-4. Implemented `capture_mhtml()` method in `AsyncPlaywrightCrawlerStrategy` class to capture MHTML via CDP
-5. Modified the crawler to capture MHTML when enabled and pass it to the result
-
-**Implementation Details:**
-- MHTML capture uses Chrome DevTools Protocol (CDP) via Playwright's CDP session API
-- The implementation waits for page to fully load before capturing MHTML content
-- Enhanced waiting for JavaScript content with requestAnimationFrame for better JS content capture
-- We ensure all browser resources are properly cleaned up after capture
-
-**Files Modified:**
-- `crawl4ai/models.py`: Added the mhtml field to CrawlResult
-- `crawl4ai/async_configs.py`: Added capture_mhtml parameter to CrawlerRunConfig
-- `crawl4ai/async_crawler_strategy.py`: Implemented MHTML capture logic
-- `crawl4ai/async_webcrawler.py`: Added mapping from AsyncCrawlResponse.mhtml_data to CrawlResult.mhtml
-
-**Testing:**
-- Created comprehensive tests in `tests/20241401/test_mhtml.py` covering:
-  - Capturing MHTML when enabled
-  - Ensuring mhtml is None when disabled explicitly
-  - Ensuring mhtml is None by default
-  - Capturing MHTML on JavaScript-enabled pages
-
-**Challenges:**
-- Had to improve page loading detection to ensure JavaScript content was fully rendered
-- Tests needed to be run independently due to Playwright browser instance management
-- Modified test expected content to match actual MHTML output
-
-**Why This Feature:**
-The MHTML capture feature allows users to capture complete web pages including all resources (CSS, images, etc.) in a single file. This is valuable for:
-1. Offline viewing of captured pages
-2. Creating permanent snapshots of web content for archival
-3. Ensuring consistent content for later analysis, even if the original site changes
-
-**Future Enhancements to Consider:**
-- Add option to save MHTML to file
-- Support for filtering what resources get included in MHTML
-- Add support for specifying MHTML capture options
-
-## [2025-04-10] Added Network Request and Console Message Capturing
-
-**Feature:** Comprehensive capturing of network requests/responses and browser console messages during crawling
-
-**Changes Made:**
-1. Added `capture_network_requests: bool = False` and `capture_console_messages: bool = False` parameters to `CrawlerRunConfig` class
-2. Added `network_requests: Optional[List[Dict[str, Any]]] = None` and `console_messages: Optional[List[Dict[str, Any]]] = None` fields to both `AsyncCrawlResponse` and `CrawlResult` models
-3. Implemented event listeners in `AsyncPlaywrightCrawlerStrategy._crawl_web()` to capture browser network events and console messages
-4. Added proper event listener cleanup in the finally block to prevent resource leaks
-5. Modified the crawler flow to pass captured data from AsyncCrawlResponse to CrawlResult
-
-**Implementation Details:**
-- Network capture uses Playwright event listeners (`request`, `response`, and `requestfailed`) to record all network activity
-- Console capture uses Playwright event listeners (`console` and `pageerror`) to record console messages and errors
-- Each network event includes metadata like URL, headers, status, and timing information
-- Each console message includes type, text content, and source location when available
-- All captured events include timestamps for chronological analysis
-- Error handling ensures even failed capture attempts won't crash the main crawling process
-
-**Files Modified:**
-- `crawl4ai/models.py`: Added new fields to AsyncCrawlResponse and CrawlResult
-- `crawl4ai/async_configs.py`: Added new configuration parameters to CrawlerRunConfig
-- `crawl4ai/async_crawler_strategy.py`: Implemented capture logic using event listeners
-- `crawl4ai/async_webcrawler.py`: Added data transfer from AsyncCrawlResponse to CrawlResult
-
-**Documentation:**
-- Created detailed documentation in `docs/md_v2/advanced/network-console-capture.md`
-- Added feature to site navigation in `mkdocs.yml`
-- Updated CrawlResult documentation in `docs/md_v2/api/crawl-result.md`
-- Created comprehensive example in `docs/examples/network_console_capture_example.py`
-
-**Testing:**
-- Created `tests/general/test_network_console_capture.py` with tests for:
-  - Verifying capture is disabled by default
-  - Testing network request capturing
-  - Testing console message capturing
-  - Ensuring both capture types can be enabled simultaneously
-  - Checking correct content is captured in expected formats
-
-**Challenges:**
-- Initial implementation had synchronous/asynchronous mismatches in event handlers
-- Needed to fix type of property access vs. method calls in handlers
-- Required careful cleanup of event listeners to prevent memory leaks
-
-**Why This Feature:**
-The network and console capture feature provides deep visibility into web page activity, enabling:
-1. Debugging complex web applications by seeing all network requests and errors
-2. Security analysis to detect unexpected third-party requests and data flows
-3. Performance profiling to identify slow-loading resources
-4. API discovery in single-page applications
-5. Comprehensive analysis of web application behavior
-
-**Future Enhancements to Consider:**
-- Option to filter captured events by type, domain, or content
-- Support for capturing response bodies (with size limits)
-- Aggregate statistics calculation for performance metrics
-- Integration with visualization tools for network waterfall analysis
-- Exporting captures in HAR format for use with external tools

LICENSE

@@ -48,22 +48,4 @@ You may add Your own copyright statement to Your modifications and may provide a
 
 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
 
-END OF TERMS AND CONDITIONS
-
----
-Attribution Requirement
-
-All distributions, publications, or public uses of this software, or derivative works based on this software, must include the following attribution:
-
-"This product includes software developed by UncleCode (https://x.com/unclecode) as part of the Crawl4AI project (https://github.com/unclecode/crawl4ai)."
-
-This attribution must be displayed in a prominent and easily accessible location, such as:
-
--   For software distributions: In a NOTICE file, README file, or equivalent documentation.
--   For publications (research papers, articles, blog posts): In the acknowledgments section or a footnote.
--   For websites/web applications: In an "About" or "Credits" section.
--   For command-line tools: In the help/usage output.
-
-This requirement ensures proper credit is given for the use of Crawl4AI and helps promote the project.
-
----
+END OF TERMS AND CONDITIONS

MANIFEST.in

@@ -1,2 +1 @@
-include requirements.txt
-recursive-include crawl4ai/js_snippet *.js
+include requirements.txt

MISSION.md

@@ -1,46 +0,0 @@
-# Mission
-
-![Mission Diagram](./docs/assets/pitch-dark.svg)
-
-### 1. The Data Capitalization Opportunity
-
-We live in an unprecedented era of digital wealth creation. Every day, individuals and enterprises generate massive amounts of valuable digital footprints across various platforms, social media channels, messenger apps, and cloud services. While people can interact with their data within these platforms, there's an immense untapped opportunity to transform this data into true capital assets. Just as physical property became a foundational element of wealth creation, personal and enterprise data has the potential to become a new form of capital on balance sheets.
-
-For individuals, this represents an opportunity to transform their digital activities into valuable assets. For enterprises, their internal communications, team discussions, and collaborative documents contain rich insights that could be structured and valued as intellectual capital. This wealth of information represents an unprecedented opportunity for value creation in the digital age.
-
-### 2. The Potential of Authentic Data
-
-While synthetic data has played a crucial role in AI development, there's an enormous untapped potential in the authentic data generated by individuals and organizations. Every message, document, and interaction contains unique insights and patterns that could enhance AI development. The challenge isn't a lack of data - it's that most authentic human-generated data remains inaccessible for productive use.
-
-By enabling willing participation in data sharing, we can unlock this vast reservoir of authentic human knowledge. This represents an opportunity to enhance AI development with diverse, real-world data that reflects the full spectrum of human experience and knowledge.
-
-## Our Pathway to Data Democracy
-
-### 1. Open-Source Foundation
-
-Our first step is creating an open-source data extraction engine that empowers developers and innovators to build tools for data structuring and organization. This foundation ensures transparency, security, and community-driven development. By making these tools openly available, we enable the technical infrastructure needed for true data ownership and capitalization.
-
-### 2. Data Capitalization Platform
-
-Building on this open-source foundation, we're developing a platform that helps individuals and enterprises transform their digital footprints into structured, valuable assets. This platform will provide the tools and frameworks needed to organize, understand, and value personal and organizational data as true capital assets.
-
-### 3. Creating a Data Marketplace
-
-The final piece is establishing a marketplace where individuals and organizations can willingly share their data assets. This creates opportunities for:
-- Individuals to earn equity, revenue, or other forms of value from their data
-- Enterprises to access diverse, high-quality data for AI development
-- Researchers to work with authentic human-generated data
-- Startups to build innovative solutions using real-world data
-
-## Economic Vision: A Shared Data Economy
-
-We envision a future where data becomes a fundamental asset class in a thriving shared economy. This transformation will democratize AI development by enabling willing participation in data sharing, ensuring that the benefits of AI advancement flow back to data creators. Just as property rights revolutionized economic systems, establishing data as a capital asset will create new opportunities for wealth creation and economic participation.
-
-This shared data economy will:
-- Enable individuals to capitalize on their digital footprints
-- Create new revenue streams for data creators
-- Provide AI developers with access to diverse, authentic data
-- Foster innovation through broader access to real-world data
-- Ensure more equitable distribution of AI's economic benefits
-
-Our vision is to facilitate this transformation from the ground up - starting with open-source tools, progressing to data capitalization platforms, and ultimately creating a thriving marketplace where data becomes a true asset class in a shared economy. This approach ensures that the future of AI is built on a foundation of authentic human knowledge, with benefits flowing back to the individuals and organizations who create and share their valuable data.

PROGRESSIVE_CRAWLING.md

@@ -1,320 +0,0 @@
-# Progressive Web Crawling with Adaptive Information Foraging
-
-## Abstract
-
-This paper presents a novel approach to web crawling that adaptively determines when sufficient information has been gathered to answer a given query. Unlike traditional exhaustive crawling methods, our Progressive Information Sufficiency (PIS) framework uses statistical measures to balance information completeness against crawling efficiency. We introduce a multi-strategy architecture supporting pure statistical, embedding-enhanced, and LLM-assisted approaches, with theoretical guarantees on convergence and practical evaluation methods using synthetic datasets.
-
-## 1. Introduction
-
-Traditional web crawling approaches follow predetermined patterns (breadth-first, depth-first) without consideration for information sufficiency. This work addresses the fundamental question: *"When do we have enough information to answer a query and similar queries in its domain?"*
-
-We formalize this as an optimal stopping problem in information foraging, introducing metrics for coverage, consistency, and saturation that enable crawlers to make intelligent decisions about when to stop crawling and which links to follow.
-
-## 2. Problem Formulation
-
-### 2.1 Definitions
-
-Let:
-- **K** = {d₁, d₂, ..., dₙ} be the current knowledge base (crawled documents)
-- **Q** be the user query
-- **L** = {l₁, l₂, ..., lₘ} be available links with preview metadata
-- **θ** be the confidence threshold for information sufficiency
-
-### 2.2 Objectives
-
-1. **Minimize** |K| (number of crawled pages)
-2. **Maximize** P(answers(Q) | K) (probability of answering Q given K)
-3. **Ensure** coverage of Q's domain (similar queries)
-
-## 3. Mathematical Framework
-
-### 3.1 Information Sufficiency Metric
-
-We define Information Sufficiency as:
-
-```
-IS(K, Q) = min(Coverage(K, Q), Consistency(K, Q), 1 - Redundancy(K)) × DomainCoverage(K, Q)
-```
-
-### 3.2 Coverage Score
-
-Coverage measures how well current knowledge covers query terms and related concepts:
-
-```
-Coverage(K, Q) = Σ(t ∈ Q) log(df(t, K) + 1) × idf(t) / |Q|
-```
-
-Where:
-- df(t, K) = document frequency of term t in knowledge base K
-- idf(t) = inverse document frequency weight
-
-### 3.3 Consistency Score
-
-Consistency measures information coherence across documents:
-
-```
-Consistency(K, Q) = 1 - Var(answers from random subsets of K)
-```
-
-This captures the principle that sufficient knowledge should provide stable answers regardless of document subset.
-
-### 3.4 Saturation Score
-
-Saturation detects diminishing returns:
-
-```
-Saturation(K) = 1 - (ΔInfo(Kₙ) / ΔInfo(K₁))
-```
-
-Where ΔInfo represents marginal information gain from the nth crawl.
-
-### 3.5 Link Value Prediction
-
-Expected information gain from uncrawled links:
-
-```
-ExpectedGain(l) = Relevance(l, Q) × Novelty(l, K) × Authority(l)
-```
-
-Components:
-- **Relevance**: BM25(preview_text, Q)
-- **Novelty**: 1 - max_similarity(preview, K)
-- **Authority**: f(url_structure, domain_metrics)
-
-## 4. Algorithmic Approach
-
-### 4.1 Progressive Crawling Algorithm
-
-```
-Algorithm: ProgressiveCrawl(start_url, query, θ)
-  K ← ∅
-  crawled ← {start_url}
-  pending ← extract_links(crawl(start_url))
-  
-  while IS(K, Q) < θ and |crawled| < max_pages:
-    candidates ← rank_by_expected_gain(pending, Q, K)
-    if max(ExpectedGain(candidates)) < min_gain:
-      break  // Diminishing returns
-    
-    to_crawl ← top_k(candidates)
-    new_docs ← parallel_crawl(to_crawl)
-    K ← K ∪ new_docs
-    crawled ← crawled ∪ to_crawl
-    pending ← extract_new_links(new_docs) - crawled
-  
-  return K
-```
-
-### 4.2 Stopping Criteria
-
-Crawling terminates when:
-1. IS(K, Q) ≥ θ (sufficient information)
-2. d(IS)/d(crawls) < ε (plateau reached)
-3. |crawled| ≥ max_pages (resource limit)
-4. max(ExpectedGain) < min_gain (no promising links)
-
-## 5. Multi-Strategy Architecture
-
-### 5.1 Strategy Pattern Design
-
-```
-AbstractStrategy
-  ├── StatisticalStrategy (no LLM, no embeddings)
-  ├── EmbeddingStrategy (with semantic similarity)
-  └── LLMStrategy (with language model assistance)
-```
-
-### 5.2 Statistical Strategy
-
-Pure statistical approach using:
-- BM25 for relevance scoring
-- Term frequency analysis for coverage
-- Graph structure for authority
-- No external models required
-
-**Advantages**: Fast, no API costs, works offline
-**Best for**: Technical documentation, specific terminology
-
-### 5.3 Embedding Strategy (Implemented)
-
-Semantic understanding through embeddings:
-- Query expansion into semantic variations
-- Coverage mapping in embedding space
-- Gap-driven link selection
-- Validation-based stopping criteria
-
-**Mathematical Framework**:
-```
-Coverage(K, Q) = mean(max_similarity(q, K) for q in Q_expanded)
-Gap(q) = 1 - max_similarity(q, K)
-LinkScore(l) = Σ(Gap(q) × relevance(l, q)) × (1 - redundancy(l, K))
-```
-
-**Key Parameters**:
-- `embedding_k_exp`: Exponential decay factor for distance-to-score mapping
-- `embedding_coverage_radius`: Distance threshold for query coverage
-- `embedding_min_confidence_threshold`: Minimum relevance threshold
-
-**Advantages**: Semantic understanding, handles ambiguity, detects irrelevance
-**Best for**: Research queries, conceptual topics, diverse content
-
-### 5.4 Progressive Enhancement Path
-
-1. **Level 0**: Statistical only (implemented)
-2. **Level 1**: + Embeddings for semantic similarity (implemented)
-3. **Level 2**: + LLM for query understanding (future)
-
-## 6. Evaluation Methodology
-
-### 6.1 Synthetic Dataset Generation
-
-Using LLM to create evaluation data:
-
-```python
-def generate_synthetic_dataset(domain_url):
-    # 1. Fully crawl domain
-    full_knowledge = exhaustive_crawl(domain_url)
-    
-    # 2. Generate answerable queries
-    queries = llm_generate_queries(full_knowledge)
-    
-    # 3. Create query variations
-    for q in queries:
-        variations = generate_variations(q)  # synonyms, sub/super queries
-    
-    return queries, variations, full_knowledge
-```
-
-### 6.2 Evaluation Metrics
-
-1. **Efficiency**: Information gained / Pages crawled
-2. **Completeness**: Answerable queries / Total queries
-3. **Redundancy**: 1 - (Unique information / Total information)
-4. **Convergence Rate**: Pages to 95% completeness
-
-### 6.3 Ablation Studies
-
-- Impact of each score component (coverage, consistency, saturation)
-- Sensitivity to threshold parameters
-- Performance across different domain types
-
-## 7. Theoretical Properties
-
-### 7.1 Convergence Guarantee
-
-**Theorem**: For finite websites, ProgressiveCrawl converges to IS(K, Q) ≥ θ or exhausts all reachable pages.
-
-**Proof sketch**: IS(K, Q) is monotonically non-decreasing with each crawl, bounded above by 1.
-
-### 7.2 Optimality
-
-Under certain assumptions about link preview accuracy:
-- Expected crawls ≤ 2 × optimal_crawls
-- Approximation ratio improves with preview quality
-
-## 8. Implementation Design
-
-### 8.1 Core Components
-
-1. **CrawlState**: Maintains crawl history and metrics
-2. **AdaptiveConfig**: Configuration parameters
-3. **CrawlStrategy**: Pluggable strategy interface
-4. **AdaptiveCrawler**: Main orchestrator
-
-### 8.2 Integration with Crawl4AI
-
-- Wraps existing AsyncWebCrawler
-- Leverages link preview functionality
-- Maintains backward compatibility
-
-### 8.3 Persistence
-
-Knowledge base serialization for:
-- Resumable crawls
-- Knowledge sharing
-- Offline analysis
-
-## 9. Future Directions
-
-### 9.1 Advanced Scoring
-
-- Temporal information value
-- Multi-query optimization
-- Active learning from user feedback
-
-### 9.2 Distributed Crawling
-
-- Collaborative knowledge building
-- Federated information sufficiency
-
-### 9.3 Domain Adaptation
-
-- Transfer learning across domains
-- Meta-learning for threshold selection
-
-## 10. Conclusion
-
-Progressive crawling with adaptive information foraging provides a principled approach to efficient web information extraction. By combining coverage, consistency, and saturation metrics, we can determine information sufficiency without ground truth labels. The multi-strategy architecture allows graceful enhancement from pure statistical to LLM-assisted approaches based on requirements and resources.
-
-## References
-
-1. Manning, C. D., Raghavan, P., & Schütze, H. (2008). Introduction to Information Retrieval. Cambridge University Press.
-
-2. Robertson, S., & Zaragoza, H. (2009). The Probabilistic Relevance Framework: BM25 and Beyond. Foundations and Trends in Information Retrieval.
-
-3. Pirolli, P., & Card, S. (1999). Information Foraging. Psychological Review, 106(4), 643-675.
-
-4. Dasgupta, S. (2005). Analysis of a greedy active learning strategy. Advances in Neural Information Processing Systems.
-
-## Appendix A: Implementation Pseudocode
-
-```python
-class StatisticalStrategy:
-    def calculate_confidence(self, state):
-        coverage = self.calculate_coverage(state)
-        consistency = self.calculate_consistency(state)
-        saturation = self.calculate_saturation(state)
-        return min(coverage, consistency, saturation)
-    
-    def calculate_coverage(self, state):
-        # BM25-based term coverage
-        term_scores = []
-        for term in state.query.split():
-            df = state.document_frequencies.get(term, 0)
-            idf = self.idf_cache.get(term, 1.0)
-            term_scores.append(log(df + 1) * idf)
-        return mean(term_scores) / max_possible_score
-    
-    def rank_links(self, state):
-        scored_links = []
-        for link in state.pending_links:
-            relevance = self.bm25_score(link.preview_text, state.query)
-            novelty = self.calculate_novelty(link, state.knowledge_base)
-            authority = self.url_authority(link.href)
-            score = relevance * novelty * authority
-            scored_links.append((link, score))
-        return sorted(scored_links, key=lambda x: x[1], reverse=True)
-```
-
-## Appendix B: Evaluation Protocol
-
-1. **Dataset Creation**:
-   - Select diverse domains (documentation, blogs, e-commerce)
-   - Generate 100 queries per domain using LLM
-   - Create query variations (5-10 per query)
-
-2. **Baseline Comparisons**:
-   - BFS crawler (depth-limited)
-   - DFS crawler (depth-limited)
-   - Random crawler
-   - Oracle (knows relevant pages)
-
-3. **Metrics Collection**:
-   - Pages crawled vs query answerability
-   - Time to sufficient confidence
-   - False positive/negative rates
-
-4. **Statistical Analysis**:
-   - ANOVA for strategy comparison
-   - Regression for parameter sensitivity
-   - Bootstrap for confidence intervals

README-first.md

@@ -1,809 +0,0 @@
-# 🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper.
-
-<div align="center">
-
-<a href="https://trendshift.io/repositories/11716" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11716" alt="unclecode%2Fcrawl4ai | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
-
-[![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
-[![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members)
-
-[![PyPI version](https://badge.fury.io/py/crawl4ai.svg)](https://badge.fury.io/py/crawl4ai)
-[![Python Version](https://img.shields.io/pypi/pyversions/crawl4ai)](https://pypi.org/project/crawl4ai/)
-[![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
-[![GitHub Sponsors](https://img.shields.io/github/sponsors/unclecode?style=flat&logo=GitHub-Sponsors&label=Sponsors&color=pink)](https://github.com/sponsors/unclecode)
-
-<p align="center">
-    <a href="https://x.com/crawl4ai">
-      <img src="https://img.shields.io/badge/Follow%20on%20X-000000?style=for-the-badge&logo=x&logoColor=white" alt="Follow on X" />
-    </a>
-    <a href="https://www.linkedin.com/company/crawl4ai">
-      <img src="https://img.shields.io/badge/Follow%20on%20LinkedIn-0077B5?style=for-the-badge&logo=linkedin&logoColor=white" alt="Follow on LinkedIn" />
-    </a>
-    <a href="https://discord.gg/jP8KfhDhyN">
-      <img src="https://img.shields.io/badge/Join%20our%20Discord-5865F2?style=for-the-badge&logo=discord&logoColor=white" alt="Join our Discord" />
-    </a>
-  </p>
-</div>
-
-Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for LLMs, AI agents, and data pipelines. Open source, flexible, and built for real-time performance, Crawl4AI empowers developers with unmatched speed, precision, and deployment ease.  
-
-[✨ Check out latest update v0.7.0](#-recent-updates)
-
-🎉 **Version 0.7.0 is now available!** The Adaptive Intelligence Update introduces groundbreaking features: Adaptive Crawling that learns website patterns, Virtual Scroll support for infinite pages, intelligent Link Preview with 3-layer scoring, Async URL Seeder for massive discovery, and significant performance improvements. [Read the release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.0.md)
-
-<details>
-<summary>🤓 <strong>My Personal Story</strong></summary>
-
-My journey with computers started in childhood when my dad, a computer scientist, introduced me to an Amstrad computer. Those early days sparked a fascination with technology, leading me to pursue computer science and specialize in NLP during my postgraduate studies. It was during this time that I first delved into web crawling, building tools to help researchers organize papers and extract information from publications a challenging yet rewarding experience that honed my skills in data extraction.
-
-Fast forward to 2023, I was working on a tool for a project and needed a crawler to convert a webpage into markdown. While exploring solutions, I found one that claimed to be open-source but required creating an account and generating an API token. Worse, it turned out to be a SaaS model charging $16, and its quality didn’t meet my standards. Frustrated, I realized this was a deeper problem. That frustration turned into turbo anger mode, and I decided to build my own solution. In just a few days, I created Crawl4AI. To my surprise, it went viral, earning thousands of GitHub stars and resonating with a global community.
-
-I made Crawl4AI open-source for two reasons. First, it’s my way of giving back to the open-source community that has supported me throughout my career. Second, I believe data should be accessible to everyone, not locked behind paywalls or monopolized by a few. Open access to data lays the foundation for the democratization of AI, a vision where individuals can train their own models and take ownership of their information. This library is the first step in a larger journey to create the best open-source data extraction and generation tool the world has ever seen, built collaboratively by a passionate community.
-
-Thank you to everyone who has supported this project, used it, and shared feedback. Your encouragement motivates me to dream even bigger. Join us, file issues, submit PRs, or spread the word. Together, we can build a tool that truly empowers people to access their own data and reshape the future of AI.
-</details>
-
-## 🧐 Why Crawl4AI?
-
-1. **Built for LLMs**: Creates smart, concise Markdown optimized for RAG and fine-tuning applications.  
-2. **Lightning Fast**: Delivers results faster with real-time, cost-efficient performance.  
-3. **Flexible Browser Control**: Offers session management, proxies, and custom hooks for seamless data access.  
-4. **Heuristic Intelligence**: Uses advanced algorithms for efficient extraction, reducing reliance on costly models.  
-5. **Open Source & Deployable**: Fully open-source with no API keys—ready for Docker and cloud integration.  
-6. **Thriving Community**: Actively maintained by a vibrant community and the #1 trending GitHub repository.
-
-## 🚀 Quick Start 
-
-1. Install Crawl4AI:
-```bash
-# Install the package
-pip install -U crawl4ai
-
-# For pre release versions
-pip install crawl4ai --pre
-
-# Run post-installation setup
-crawl4ai-setup
-
-# Verify your installation
-crawl4ai-doctor
-```
-
-If you encounter any browser-related issues, you can install them manually:
-```bash
-python -m playwright install --with-deps chromium
-```
-
-2. Run a simple web crawl with Python:
-```python
-import asyncio
-from crawl4ai import *
-
-async def main():
-    async with AsyncWebCrawler() as crawler:
-        result = await crawler.arun(
-            url="https://www.nbcnews.com/business",
-        )
-        print(result.markdown)
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-3. Or use the new command-line interface:
-```bash
-# Basic crawl with markdown output
-crwl https://www.nbcnews.com/business -o markdown
-
-# Deep crawl with BFS strategy, max 10 pages
-crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10
-
-# Use LLM extraction with a specific question
-crwl https://www.example.com/products -q "Extract all product prices"
-```
-
-## ✨ Features 
-
-<details>
-<summary>📝 <strong>Markdown Generation</strong></summary>
-
-- 🧹 **Clean Markdown**: Generates clean, structured Markdown with accurate formatting.
-- 🎯 **Fit Markdown**: Heuristic-based filtering to remove noise and irrelevant parts for AI-friendly processing.
-- 🔗 **Citations and References**: Converts page links into a numbered reference list with clean citations.
-- 🛠️ **Custom Strategies**: Users can create their own Markdown generation strategies tailored to specific needs.
-- 📚 **BM25 Algorithm**: Employs BM25-based filtering for extracting core information and removing irrelevant content. 
-</details>
-
-<details>
-<summary>📊 <strong>Structured Data Extraction</strong></summary>
-
-- 🤖 **LLM-Driven Extraction**: Supports all LLMs (open-source and proprietary) for structured data extraction.
-- 🧱 **Chunking Strategies**: Implements chunking (topic-based, regex, sentence-level) for targeted content processing.
-- 🌌 **Cosine Similarity**: Find relevant content chunks based on user queries for semantic extraction.
-- 🔎 **CSS-Based Extraction**: Fast schema-based data extraction using XPath and CSS selectors.
-- 🔧 **Schema Definition**: Define custom schemas for extracting structured JSON from repetitive patterns.
-
-</details>
-
-<details>
-<summary>🌐 <strong>Browser Integration</strong></summary>
-
-- 🖥️ **Managed Browser**: Use user-owned browsers with full control, avoiding bot detection.
-- 🔄 **Remote Browser Control**: Connect to Chrome Developer Tools Protocol for remote, large-scale data extraction.
-- 👤 **Browser Profiler**: Create and manage persistent profiles with saved authentication states, cookies, and settings.
-- 🔒 **Session Management**: Preserve browser states and reuse them for multi-step crawling.
-- 🧩 **Proxy Support**: Seamlessly connect to proxies with authentication for secure access.
-- ⚙️ **Full Browser Control**: Modify headers, cookies, user agents, and more for tailored crawling setups.
-- 🌍 **Multi-Browser Support**: Compatible with Chromium, Firefox, and WebKit.
-- 📐 **Dynamic Viewport Adjustment**: Automatically adjusts the browser viewport to match page content, ensuring complete rendering and capturing of all elements.
-
-</details>
-
-<details>
-<summary>🔎 <strong>Crawling & Scraping</strong></summary>
-
-- 🖼️ **Media Support**: Extract images, audio, videos, and responsive image formats like `srcset` and `picture`.
-- 🚀 **Dynamic Crawling**: Execute JS and wait for async or sync for dynamic content extraction.
-- 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis.
-- 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`).
-- 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content.
-- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior.
-- 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches.
-- 📄 **Metadata Extraction**: Retrieve structured metadata from web pages.
-- 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content.
-- 🕵️ **Lazy Load Handling**: Waits for images to fully load, ensuring no content is missed due to lazy loading.
-- 🔄 **Full-Page Scanning**: Simulates scrolling to load and capture all dynamic content, perfect for infinite scroll pages.
-
-</details>
-
-<details>
-<summary>🚀 <strong>Deployment</strong></summary>
-
-- 🐳 **Dockerized Setup**: Optimized Docker image with FastAPI server for easy deployment.
-- 🔑 **Secure Authentication**: Built-in JWT token authentication for API security.
-- 🔄 **API Gateway**: One-click deployment with secure token authentication for API-based workflows.
-- 🌐 **Scalable Architecture**: Designed for mass-scale production and optimized server performance.
-- ☁️ **Cloud Deployment**: Ready-to-deploy configurations for major cloud platforms.
-
-</details>
-
-<details>
-<summary>🎯 <strong>Additional Features</strong></summary>
-
-- 🕶️ **Stealth Mode**: Avoid bot detection by mimicking real users.
-- 🏷️ **Tag-Based Content Extraction**: Refine crawling based on custom tags, headers, or metadata.
-- 🔗 **Link Analysis**: Extract and analyze all links for detailed data exploration.
-- 🛡️ **Error Handling**: Robust error management for seamless execution.
-- 🔐 **CORS & Static Serving**: Supports filesystem-based caching and cross-origin requests.
-- 📖 **Clear Documentation**: Simplified and updated guides for onboarding and advanced usage.
-- 🙌 **Community Recognition**: Acknowledges contributors and pull requests for transparency.
-
-</details>
-
-## Try it Now!
-
-✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing)
-
-✨ Visit our [Documentation Website](https://docs.crawl4ai.com/)
-
-## Installation 🛠️
-
-Crawl4AI offers flexible installation options to suit various use cases. You can install it as a Python package or use Docker.
-
-<details>
-<summary>🐍 <strong>Using pip</strong></summary>
-
-Choose the installation option that best fits your needs:
-
-### Basic Installation
-
-For basic web crawling and scraping tasks:
-
-```bash
-pip install crawl4ai
-crawl4ai-setup # Setup the browser
-```
-
-By default, this will install the asynchronous version of Crawl4AI, using Playwright for web crawling.
-
-👉 **Note**: When you install Crawl4AI, the `crawl4ai-setup` should automatically install and set up Playwright. However, if you encounter any Playwright-related errors, you can manually install it using one of these methods:
-
-1. Through the command line:
-
-   ```bash
-   playwright install
-   ```
-
-2. If the above doesn't work, try this more specific command:
-
-   ```bash
-   python -m playwright install chromium
-   ```
-
-This second method has proven to be more reliable in some cases.
-
----
-
-### Installation with Synchronous Version
-
-The sync version is deprecated and will be removed in future versions. If you need the synchronous version using Selenium:
-
-```bash
-pip install crawl4ai[sync]
-```
-
----
-
-### Development Installation
-
-For contributors who plan to modify the source code:
-
-```bash
-git clone https://github.com/unclecode/crawl4ai.git
-cd crawl4ai
-pip install -e .                    # Basic installation in editable mode
-```
-
-Install optional features:
-
-```bash
-pip install -e ".[torch]"           # With PyTorch features
-pip install -e ".[transformer]"     # With Transformer features
-pip install -e ".[cosine]"          # With cosine similarity features
-pip install -e ".[sync]"            # With synchronous crawling (Selenium)
-pip install -e ".[all]"             # Install all optional features
-```
-
-</details>
-
-<details>
-<summary>🐳 <strong>Docker Deployment</strong></summary>
-
-> 🚀 **Now Available!** Our completely redesigned Docker implementation is here! This new solution makes deployment more efficient and seamless than ever.
-
-### New Docker Features
-
-The new Docker implementation includes:
-- **Browser pooling** with page pre-warming for faster response times
-- **Interactive playground** to test and generate request code
-- **MCP integration** for direct connection to AI tools like Claude Code
-- **Comprehensive API endpoints** including HTML extraction, screenshots, PDF generation, and JavaScript execution
-- **Multi-architecture support** with automatic detection (AMD64/ARM64)
-- **Optimized resources** with improved memory management
-
-### Getting Started
-
-```bash
-# Pull and run the latest release candidate
-docker pull unclecode/crawl4ai:0.7.0
-docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:0.7.0
-
-# Visit the playground at http://localhost:11235/playground
-```
-
-For complete documentation, see our [Docker Deployment Guide](https://docs.crawl4ai.com/core/docker-deployment/).
-
-</details>
-
----
-
-### Quick Test
-
-Run a quick test (works for both Docker options):
-
-```python
-import requests
-
-# Submit a crawl job
-response = requests.post(
-    "http://localhost:11235/crawl",
-    json={"urls": ["https://example.com"], "priority": 10}
-)
-if response.status_code == 200:
-    print("Crawl job submitted successfully.")
-    
-if "results" in response.json():
-    results = response.json()["results"]
-    print("Crawl job completed. Results:")
-    for result in results:
-        print(result)
-else:
-    task_id = response.json()["task_id"]
-    print(f"Crawl job submitted. Task ID:: {task_id}")
-    result = requests.get(f"http://localhost:11235/task/{task_id}")
-```
-
-For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://docs.crawl4ai.com/basic/docker-deployment/).
-
-</details>
-
-
-## 🔬 Advanced Usage Examples 🔬
-
-You can check the project structure in the directory [docs/examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples). Over there, you can find a variety of examples; here, some popular examples are shared.
-
-<details>
-<summary>📝 <strong>Heuristic Markdown Generation with Clean and Fit Markdown</strong></summary>
-
-```python
-import asyncio
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
-from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
-
-async def main():
-    browser_config = BrowserConfig(
-        headless=True,  
-        verbose=True,
-    )
-    run_config = CrawlerRunConfig(
-        cache_mode=CacheMode.ENABLED,
-        markdown_generator=DefaultMarkdownGenerator(
-            content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed", min_word_threshold=0)
-        ),
-        # markdown_generator=DefaultMarkdownGenerator(
-        #     content_filter=BM25ContentFilter(user_query="WHEN_WE_FOCUS_BASED_ON_A_USER_QUERY", bm25_threshold=1.0)
-        # ),
-    )
-    
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(
-            url="https://docs.micronaut.io/4.7.6/guide/",
-            config=run_config
-        )
-        print(len(result.markdown.raw_markdown))
-        print(len(result.markdown.fit_markdown))
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-</details>
-
-<details>
-<summary>🖥️ <strong>Executing JavaScript & Extract Structured Data without LLMs</strong></summary>
-
-```python
-import asyncio
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai import JsonCssExtractionStrategy
-import json
-
-async def main():
-    schema = {
-    "name": "KidoCode Courses",
-    "baseSelector": "section.charge-methodology .w-tab-content > div",
-    "fields": [
-        {
-            "name": "section_title",
-            "selector": "h3.heading-50",
-            "type": "text",
-        },
-        {
-            "name": "section_description",
-            "selector": ".charge-content",
-            "type": "text",
-        },
-        {
-            "name": "course_name",
-            "selector": ".text-block-93",
-            "type": "text",
-        },
-        {
-            "name": "course_description",
-            "selector": ".course-content-text",
-            "type": "text",
-        },
-        {
-            "name": "course_icon",
-            "selector": ".image-92",
-            "type": "attribute",
-            "attribute": "src"
-        }
-    }
-}
-
-    extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
-
-    browser_config = BrowserConfig(
-        headless=False,
-        verbose=True
-    )
-    run_config = CrawlerRunConfig(
-        extraction_strategy=extraction_strategy,
-        js_code=["""(async () => {const tabs = document.querySelectorAll("section.charge-methodology .tabs-menu-3 > div");for(let tab of tabs) {tab.scrollIntoView();tab.click();await new Promise(r => setTimeout(r, 500));}})();"""],
-        cache_mode=CacheMode.BYPASS
-    )
-        
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        
-        result = await crawler.arun(
-            url="https://www.kidocode.com/degrees/technology",
-            config=run_config
-        )
-
-        companies = json.loads(result.extracted_content)
-        print(f"Successfully extracted {len(companies)} companies")
-        print(json.dumps(companies[0], indent=2))
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-</details>
-
-<details>
-<summary>📚 <strong>Extracting Structured Data with LLMs</strong></summary>
-
-```python
-import os
-import asyncio
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LLMConfig
-from crawl4ai import LLMExtractionStrategy
-from pydantic import BaseModel, Field
-
-class OpenAIModelFee(BaseModel):
-    model_name: str = Field(..., description="Name of the OpenAI model.")
-    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
-    output_fee: str = Field(..., description="Fee for output token for the OpenAI model.")
-
-async def main():
-    browser_config = BrowserConfig(verbose=True)
-    run_config = CrawlerRunConfig(
-        word_count_threshold=1,
-        extraction_strategy=LLMExtractionStrategy(
-            # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2
-            # provider="ollama/qwen2", api_token="no-token", 
-            llm_config = LLMConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')), 
-            schema=OpenAIModelFee.schema(),
-            extraction_type="schema",
-            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
-            Do not miss any models in the entire content. One extracted model JSON format should look like this: 
-            {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""
-        ),            
-        cache_mode=CacheMode.BYPASS,
-    )
-    
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        result = await crawler.arun(
-            url='https://openai.com/api/pricing/',
-            config=run_config
-        )
-        print(result.extracted_content)
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-</details>
-
-<details>
-<summary>🤖 <strong>Using Your own Browser with Custom User Profile</strong></summary>
-
-```python
-import os, sys
-from pathlib import Path
-import asyncio, time
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-
-async def test_news_crawl():
-    # Create a persistent user data directory
-    user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
-    os.makedirs(user_data_dir, exist_ok=True)
-
-    browser_config = BrowserConfig(
-        verbose=True,
-        headless=True,
-        user_data_dir=user_data_dir,
-        use_persistent_context=True,
-    )
-    run_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS
-    )
-    
-    async with AsyncWebCrawler(config=browser_config) as crawler:
-        url = "ADDRESS_OF_A_CHALLENGING_WEBSITE"
-        
-        result = await crawler.arun(
-            url,
-            config=run_config,
-            magic=True,
-        )
-        
-        print(f"Successfully crawled {url}")
-        print(f"Content length: {len(result.markdown)}")
-```
-
-</details>
-
-## ✨ Recent Updates
-
-### Version 0.7.0 Release Highlights - The Adaptive Intelligence Update
-
-- **🧠 Adaptive Crawling**: Your crawler now learns and adapts to website patterns automatically:
-  ```python
-  config = AdaptiveConfig(
-      confidence_threshold=0.7, # Min confidence to stop crawling
-      max_depth=5, # Maximum crawl depth
-      max_pages=20, # Maximum number of pages to crawl
-      strategy="statistical"
-  )
-  
-  async with AsyncWebCrawler() as crawler:
-      adaptive_crawler = AdaptiveCrawler(crawler, config)
-      state = await adaptive_crawler.digest(
-          start_url="https://news.example.com",
-          query="latest news content"
-      )
-  # Crawler learns patterns and improves extraction over time
-  ```
-
-- **🌊 Virtual Scroll Support**: Complete content extraction from infinite scroll pages:
-  ```python
-  scroll_config = VirtualScrollConfig(
-      container_selector="[data-testid='feed']",
-      scroll_count=20,
-      scroll_by="container_height",
-      wait_after_scroll=1.0
-  )
-  
-  result = await crawler.arun(url, config=CrawlerRunConfig(
-      virtual_scroll_config=scroll_config
-  ))
-  ```
-
-- **🔗 Intelligent Link Analysis**: 3-layer scoring system for smart link prioritization:
-  ```python
-  link_config = LinkPreviewConfig(
-      query="machine learning tutorials",
-      score_threshold=0.3,
-      concurrent_requests=10
-  )
-  
-  result = await crawler.arun(url, config=CrawlerRunConfig(
-      link_preview_config=link_config,
-      score_links=True
-  ))
-  # Links ranked by relevance and quality
-  ```
-
-- **🎣 Async URL Seeder**: Discover thousands of URLs in seconds:
-  ```python
-  seeder = AsyncUrlSeeder(SeedingConfig(
-      source="sitemap+cc",
-      pattern="*/blog/*",
-      query="python tutorials",
-      score_threshold=0.4
-  ))
-  
-  urls = await seeder.discover("https://example.com")
-  ```
-
-- **⚡ Performance Boost**: Up to 3x faster with optimized resource handling and memory efficiency
-
-Read the full details in our [0.7.0 Release Notes](https://docs.crawl4ai.com/blog/release-v0.7.0) or check the [CHANGELOG](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md).
-
-## Version Numbering in Crawl4AI
-
-Crawl4AI follows standard Python version numbering conventions (PEP 440) to help users understand the stability and features of each release.
-
-### Version Numbers Explained
-
-Our version numbers follow this pattern: `MAJOR.MINOR.PATCH` (e.g., 0.4.3)
-
-#### Pre-release Versions
-We use different suffixes to indicate development stages:
-
-- `dev` (0.4.3dev1): Development versions, unstable
-- `a` (0.4.3a1): Alpha releases, experimental features
-- `b` (0.4.3b1): Beta releases, feature complete but needs testing
-- `rc` (0.4.3): Release candidates, potential final version
-
-#### Installation
-- Regular installation (stable version):
-  ```bash
-  pip install -U crawl4ai
-  ```
-
-- Install pre-release versions:
-  ```bash
-  pip install crawl4ai --pre
-  ```
-
-- Install specific version:
-  ```bash
-  pip install crawl4ai==0.4.3b1
-  ```
-
-#### Why Pre-releases?
-We use pre-releases to:
-- Test new features in real-world scenarios
-- Gather feedback before final releases
-- Ensure stability for production users
-- Allow early adopters to try new features
-
-For production environments, we recommend using the stable version. For testing new features, you can opt-in to pre-releases using the `--pre` flag.
-
-## 📖 Documentation & Roadmap 
-
-> 🚨 **Documentation Update Alert**: We're undertaking a major documentation overhaul next week to reflect recent updates and improvements. Stay tuned for a more comprehensive and up-to-date guide!
-
-For current documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://docs.crawl4ai.com/).
-
-To check our development plans and upcoming features, visit our [Roadmap](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md).
-
-<details>
-<summary>📈 <strong>Development TODOs</strong></summary>
-
-- [x] 0. Graph Crawler: Smart website traversal using graph search algorithms for comprehensive nested page extraction
-- [ ] 1. Question-Based Crawler: Natural language driven web discovery and content extraction
-- [ ] 2. Knowledge-Optimal Crawler: Smart crawling that maximizes knowledge while minimizing data extraction
-- [ ] 3. Agentic Crawler: Autonomous system for complex multi-step crawling operations
-- [ ] 4. Automated Schema Generator: Convert natural language to extraction schemas
-- [ ] 5. Domain-Specific Scrapers: Pre-configured extractors for common platforms (academic, e-commerce)
-- [ ] 6. Web Embedding Index: Semantic search infrastructure for crawled content
-- [ ] 7. Interactive Playground: Web UI for testing, comparing strategies with AI assistance
-- [ ] 8. Performance Monitor: Real-time insights into crawler operations
-- [ ] 9. Cloud Integration: One-click deployment solutions across cloud providers
-- [ ] 10. Sponsorship Program: Structured support system with tiered benefits
-- [ ] 11. Educational Content: "How to Crawl" video series and interactive tutorials
-
-</details>
-
-## 🤝 Contributing 
-
-We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTORS.md) for more information.
-
-I'll help modify the license section with badges. For the halftone effect, here's a version with it:
-
-Here's the updated license section:
-
-## 📄 License & Attribution
-
-This project is licensed under the Apache License 2.0, attribution is recommended via the badges below. See the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE) file for details.
-
-### Attribution Requirements
-When using Crawl4AI, you must include one of the following attribution methods:
-
-#### 1. Badge Attribution (Recommended)
-Add one of these badges to your README, documentation, or website:
-
-| Theme | Badge |
-|-------|-------|
-| **Disco Theme (Animated)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/></a> |
-| **Night Theme (Dark with Neon)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/></a> |
-| **Dark Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/></a> |
-| **Light Theme (Classic)** | <a href="https://github.com/unclecode/crawl4ai"><img src="./docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/></a> |
- 
-
-HTML code for adding the badges:
-```html
-<!-- Disco Theme (Animated) -->
-<a href="https://github.com/unclecode/crawl4ai">
-  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-disco.svg" alt="Powered by Crawl4AI" width="200"/>
-</a>
-
-<!-- Night Theme (Dark with Neon) -->
-<a href="https://github.com/unclecode/crawl4ai">
-  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-night.svg" alt="Powered by Crawl4AI" width="200"/>
-</a>
-
-<!-- Dark Theme (Classic) -->
-<a href="https://github.com/unclecode/crawl4ai">
-  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-dark.svg" alt="Powered by Crawl4AI" width="200"/>
-</a>
-
-<!-- Light Theme (Classic) -->
-<a href="https://github.com/unclecode/crawl4ai">
-  <img src="https://raw.githubusercontent.com/unclecode/crawl4ai/main/docs/assets/powered-by-light.svg" alt="Powered by Crawl4AI" width="200"/>
-</a>
-
-<!-- Simple Shield Badge -->
-<a href="https://github.com/unclecode/crawl4ai">
-  <img src="https://img.shields.io/badge/Powered%20by-Crawl4AI-blue?style=flat-square" alt="Powered by Crawl4AI"/>
-</a>
-```
-
-#### 2. Text Attribution
-Add this line to your documentation:
-```
-This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web data extraction.
-```
-
-## 📚 Citation
-
-If you use Crawl4AI in your research or project, please cite:
-
-```bibtex
-@software{crawl4ai2024,
-  author = {UncleCode},
-  title = {Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper},
-  year = {2024},
-  publisher = {GitHub},
-  journal = {GitHub Repository},
-  howpublished = {\url{https://github.com/unclecode/crawl4ai}},
-  commit = {Please use the commit hash you're working with}
-}
-```
-
-Text citation format:
-```
-UncleCode. (2024). Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper [Computer software]. 
-GitHub. https://github.com/unclecode/crawl4ai
-```
-
-## 📧 Contact 
-
-For questions, suggestions, or feedback, feel free to reach out:
-
-- GitHub: [unclecode](https://github.com/unclecode)
-- Twitter: [@unclecode](https://twitter.com/unclecode)
-- Website: [crawl4ai.com](https://crawl4ai.com)
-
-Happy Crawling! 🕸️🚀
-
-## 💖 Support Crawl4AI
-
-> 🎉 **Sponsorship Program Just Launched!** Be among the first 50 **Founding Sponsors** and get permanent recognition in our Hall of Fame!
-
-Crawl4AI is the #1 trending open-source web crawler with 51K+ stars. Your support ensures we stay independent, innovative, and free forever.
-
-<div align="center">
-
-[![Become a Sponsor](https://img.shields.io/badge/Become%20a%20Sponsor-pink?style=for-the-badge&logo=github-sponsors&logoColor=white)](https://github.com/sponsors/unclecode)
-[![Current Sponsors](https://img.shields.io/github/sponsors/unclecode?style=for-the-badge&logo=github&label=Current%20Sponsors&color=green)](https://github.com/sponsors/unclecode)
-
-</div>
-
-### 🤝 Sponsorship Tiers
-
-- **🌱 Believer ($5/mo)**: Join the movement for data democratization
-- **🚀 Builder ($50/mo)**: Get priority support and early feature access  
-- **💼 Growing Team ($500/mo)**: Bi-weekly syncs and optimization help
-- **🏢 Data Infrastructure Partner ($2000/mo)**: Full partnership with dedicated support
-
-**Why sponsor?** Every tier includes real benefits. No more rate-limited APIs. Own your data pipeline. Build data sovereignty together.
-
-[View All Tiers & Benefits →](https://github.com/sponsors/unclecode)
-
-### 🏆 Our Sponsors
-
-#### 👑 Founding Sponsors (First 50)
-*Be part of history - [Become a Founding Sponsor](https://github.com/sponsors/unclecode)*
-
-<!-- Founding sponsors will be permanently recognized here -->
-
-#### Current Sponsors
-Thank you to all our sponsors who make this project possible!
-
-<!-- Sponsors will be automatically added here -->
-
-## 🗾 Mission
-
-Our mission is to unlock the value of personal and enterprise data by transforming digital footprints into structured, tradeable assets. Crawl4AI empowers individuals and organizations with open-source tools to extract and structure data, fostering a shared data economy.  
-
-We envision a future where AI is powered by real human knowledge, ensuring data creators directly benefit from their contributions. By democratizing data and enabling ethical sharing, we are laying the foundation for authentic AI advancement.
-
-<details>
-<summary>🔑 <strong>Key Opportunities</strong></summary>
- 
-- **Data Capitalization**: Transform digital footprints into measurable, valuable assets.  
-- **Authentic AI Data**: Provide AI systems with real human insights.  
-- **Shared Economy**: Create a fair data marketplace that benefits data creators.  
-
-</details>
-
-<details>
-<summary>🚀 <strong>Development Pathway</strong></summary>
-
-1. **Open-Source Tools**: Community-driven platforms for transparent data extraction.  
-2. **Digital Asset Structuring**: Tools to organize and value digital knowledge.  
-3. **Ethical Data Marketplace**: A secure, fair platform for exchanging structured data.  
-
-For more details, see our [full mission statement](./MISSION.md).
-</details>
-
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

README.sync.md

@@ -0,0 +1,244 @@
+# Crawl4AI v0.2.77 🕷️🤖
+
+[![GitHub Stars](https://img.shields.io/github/stars/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/stargazers)
+[![GitHub Forks](https://img.shields.io/github/forks/unclecode/crawl4ai?style=social)](https://github.com/unclecode/crawl4ai/network/members)
+[![GitHub Issues](https://img.shields.io/github/issues/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/issues)
+[![GitHub Pull Requests](https://img.shields.io/github/issues-pr/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/pulls)
+[![License](https://img.shields.io/github/license/unclecode/crawl4ai)](https://github.com/unclecode/crawl4ai/blob/main/LICENSE)
+
+Crawl4AI simplifies web crawling and data extraction, making it accessible for large language models (LLMs) and AI applications. 🆓🌐
+
+#### [v0.2.77] - 2024-08-02
+
+Major improvements in functionality, performance, and cross-platform compatibility! 🚀
+
+- 🐳 **Docker enhancements**:
+  - Significantly improved Dockerfile for easy installation on Linux, Mac, and Windows.
+- 🌐 **Official Docker Hub image**:
+  - Launched our first official image on Docker Hub for streamlined deployment (unclecode/crawl4ai).
+- 🔧 **Selenium upgrade**:
+  - Removed dependency on ChromeDriver, now using Selenium's built-in capabilities for better compatibility.
+- 🖼️ **Image description**:
+  - Implemented ability to generate textual descriptions for extracted images from web pages.
+- ⚡ **Performance boost**:
+  - Various improvements to enhance overall speed and performance.
+  
+## Try it Now!
+
+✨ Play around with this [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1sJPAmeLj5PMrg2VgOwMJ2ubGIcK0cJeX?usp=sharing)
+
+✨ visit our [Documentation Website](https://crawl4ai.com/mkdocs/)
+
+✨ Check [Demo](https://crawl4ai.com/mkdocs/demo)
+
+## Features ✨
+
+- 🆓 Completely free and open-source
+- 🤖 LLM-friendly output formats (JSON, cleaned HTML, markdown)
+- 🌍 Supports crawling multiple URLs simultaneously
+- 🎨 Extracts and returns all media tags (Images, Audio, and Video)
+- 🔗 Extracts all external and internal links
+- 📚 Extracts metadata from the page
+- 🔄 Custom hooks for authentication, headers, and page modifications before crawling
+- 🕵️ User-agent customization
+- 🖼️ Takes screenshots of the page
+- 📜 Executes multiple custom JavaScripts before crawling
+- 📚 Various chunking strategies: topic-based, regex, sentence, and more
+- 🧠 Advanced extraction strategies: cosine clustering, LLM, and more
+- 🎯 CSS selector support
+- 📝 Passes instructions/keywords to refine extraction
+
+# Crawl4AI
+
+## 🌟 Shoutout to Contributors of v0.2.77!
+
+A big thank you to the amazing contributors who've made this release possible:
+
+- [@aravindkarnam](https://github.com/aravindkarnam) for the new image description feature
+- [@FractalMind](https://github.com/FractalMind) for our official Docker Hub image
+- [@ketonkss4](https://github.com/ketonkss4) for helping streamline our Selenium setup
+
+Your contributions are driving Crawl4AI forward! 🚀
+
+## Cool Examples 🚀
+
+### Quick Start
+
+```python
+from crawl4ai import WebCrawler
+
+# Create an instance of WebCrawler
+crawler = WebCrawler()
+
+# Warm up the crawler (load necessary models)
+crawler.warmup()
+
+# Run the crawler on a URL
+result = crawler.run(url="https://www.nbcnews.com/business")
+
+# Print the extracted content
+print(result.markdown)
+```
+
+## How to install 🛠 
+
+### Using pip 🐍
+```bash
+virtualenv venv
+source venv/bin/activate
+pip install "crawl4ai @ git+https://github.com/unclecode/crawl4ai.git"
+```
+
+### Using Docker 🐳
+
+```bash
+# For Mac users (M1/M2)
+# docker build --platform linux/amd64 -t crawl4ai .
+docker build -t crawl4ai .
+docker run -d -p 8000:80 crawl4ai
+```
+
+### Using Docker Hub 🐳
+
+```bash
+docker pull unclecode/crawl4ai:latest
+docker run -d -p 8000:80 unclecode/crawl4ai:latest
+```
+
+
+## Speed-First Design 🚀
+
+Perhaps the most important design principle for this library is speed. We need to ensure it can handle many links and resources in parallel as quickly as possible. By combining this speed with fast LLMs like Groq, the results will be truly amazing.
+
+```python
+import time
+from crawl4ai.web_crawler import WebCrawler
+crawler = WebCrawler()
+crawler.warmup()
+
+start = time.time()
+url = r"https://www.nbcnews.com/business"
+result = crawler.run( url, word_count_threshold=10, bypass_cache=True)
+end = time.time()
+print(f"Time taken: {end - start}")
+```
+
+Let's take a look the calculated time for the above code snippet:
+
+```bash
+[LOG] 🚀 Crawling done, success: True, time taken: 1.3623387813568115 seconds
+[LOG] 🚀 Content extracted, success: True, time taken: 0.05715131759643555 seconds
+[LOG] 🚀 Extraction, time taken: 0.05750393867492676 seconds.
+Time taken: 1.439958095550537
+```
+Fetching the content from the page took 1.3623 seconds, and extracting the content took 0.0575 seconds. 🚀
+
+### Extract Structured Data from Web Pages 📊
+
+Crawl all OpenAI models and their fees from the official page.
+
+```python
+import os
+from crawl4ai import WebCrawler
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+from pydantic import BaseModel, Field
+
+class OpenAIModelFee(BaseModel):
+    model_name: str = Field(..., description="Name of the OpenAI model.")
+    input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+    output_fee: str = Field(..., description="Fee for output token ßfor the OpenAI model.")
+
+url = 'https://openai.com/api/pricing/'
+crawler = WebCrawler()
+crawler.warmup()
+
+result = crawler.run(
+        url=url,
+        word_count_threshold=1,
+        extraction_strategy= LLMExtractionStrategy(
+            provider= "openai/gpt-4o", api_token = os.getenv('OPENAI_API_KEY'), 
+            schema=OpenAIModelFee.schema(),
+            extraction_type="schema",
+            instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. 
+            Do not miss any models in the entire content. One extracted model JSON format should look like this: 
+            {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""
+        ),            
+        bypass_cache=True,
+    )
+
+print(result.extracted_content)
+```
+
+### Execute JS, Filter Data with CSS Selector, and Clustering
+
+```python
+from crawl4ai import WebCrawler
+from crawl4ai.chunking_strategy import CosineStrategy
+
+js_code = ["const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"]
+
+crawler = WebCrawler()
+crawler.warmup()
+
+result = crawler.run(
+    url="https://www.nbcnews.com/business",
+    js=js_code,
+    css_selector="p",
+    extraction_strategy=CosineStrategy(semantic_filter="technology")
+)
+
+print(result.extracted_content)
+```
+
+### Extract Structured Data from Web Pages With Proxy and BaseUrl
+
+```python
+from crawl4ai import WebCrawler
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+def create_crawler():
+    crawler = WebCrawler(verbose=True, proxy="http://127.0.0.1:7890")
+    crawler.warmup()
+    return crawler
+
+crawler = create_crawler()
+
+crawler.warmup()
+
+result = crawler.run(
+    url="https://www.nbcnews.com/business",
+    extraction_strategy=LLMExtractionStrategy(
+        provider="openai/gpt-4o",
+        api_token="sk-",
+        base_url="https://api.openai.com/v1"
+    )
+)
+
+print(result.markdown)
+```
+
+## Documentation 📚
+
+For detailed documentation, including installation instructions, advanced features, and API reference, visit our [Documentation Website](https://crawl4ai.com/mkdocs/).
+
+## Contributing 🤝
+
+We welcome contributions from the open-source community. Check out our [contribution guidelines](https://github.com/unclecode/crawl4ai/blob/main/CONTRIBUTING.md) for more information.
+
+## License 📄
+
+Crawl4AI is released under the [Apache 2.0 License](https://github.com/unclecode/crawl4ai/blob/main/LICENSE).
+
+## Contact 📧
+
+For questions, suggestions, or feedback, feel free to reach out:
+
+- GitHub: [unclecode](https://github.com/unclecode)
+- Twitter: [@unclecode](https://twitter.com/unclecode)
+- Website: [crawl4ai.com](https://crawl4ai.com)
+
+Happy Crawling! 🕸️🚀
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

ROADMAP.md

@@ -1,503 +0,0 @@
-# Crawl4AI Strategic Roadmap
-
-```mermaid
-%%{init: {'themeVariables': { 'fontSize': '14px'}}}%%
-graph TD
-    subgraph A1[Advanced Crawling Systems 🔧]
-        A["`
-        • Graph Crawler ✓
-        • Question-Based Crawler
-        • Knowledge-Optimal Crawler
-        • Agentic Crawler
-        `"]
-    end
-
-    subgraph A2[Specialized Features 🛠️]
-        B["`
-        • Automated Schema Generator
-        • Domain-Specific Scrapers
-        • 
-        • 
-        `"]
-    end
-
-    subgraph A3[Development Tools 🔨]
-        C["`
-        • Interactive Playground
-        • Performance Monitor
-        • Cloud Integration
-        • 
-        `"]
-    end
-
-    subgraph A4[Community & Growth 🌱]
-        D["`
-        • Sponsorship Program
-        • Educational Content
-        • 
-        • 
-        `"]
-    end
-
-    classDef default fill:#f9f9f9,stroke:#333,stroke-width:2px
-    classDef section fill:#f0f0f0,stroke:#333,stroke-width:4px,rx:10
-    class A1,A2,A3,A4 section
-
-    %% Layout hints
-    A1 --> A2[" "]
-    A3 --> A4[" "]
-    linkStyle 0,1 stroke:none
-```
-
-Crawl4AI is evolving to provide more intelligent, efficient, and versatile web crawling capabilities. This roadmap outlines the key developments and features planned for the project, organized into strategic sections that build upon our current foundation.
-
-## 1. Advanced Crawling Systems 🔧
-
-This section introduces three powerful crawling systems that extend Crawl4AI's capabilities from basic web crawling to intelligent, purpose-driven data extraction.
-
-### 1.1 Question-Based Crawler
-The Question-Based Crawler enhances our core engine by enabling automatic discovery and extraction of relevant web content based on natural language questions.
-
-Key Features:
-- SerpiAPI integration for intelligent web search
-- Relevancy scoring for search results
-- Automatic URL discovery and prioritization
-- Cross-source validation
-
-```python
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.discovery import QuestionBasedDiscovery
-
-async with AsyncWebCrawler() as crawler:
-    discovery = QuestionBasedDiscovery(crawler)
-    results = await discovery.arun(
-        question="What are the system requirements for major cloud providers' GPU instances?",
-        max_urls=5,
-        relevance_threshold=0.7
-    )
-    
-    for result in results:
-        print(f"Source: {result.url} (Relevance: {result.relevance_score})")
-        print(f"Content: {result.markdown}\n")
-```
-
-### 1.2 Knowledge-Optimal Crawler
-An intelligent crawling system that solves the optimization problem of minimizing data extraction while maximizing knowledge acquisition for specific objectives.
-
-Key Features:
-- Smart content prioritization
-- Minimal data extraction for maximum knowledge
-- Probabilistic relevance assessment
-- Objective-driven crawling paths
-
-```python
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.optimization import KnowledgeOptimizer
-
-async with AsyncWebCrawler() as crawler:
-    optimizer = KnowledgeOptimizer(
-        objective="Understand GPU instance pricing and limitations across cloud providers",
-        required_knowledge=[
-            "pricing structure",
-            "GPU specifications",
-            "usage limits",
-            "availability zones"
-        ],
-        confidence_threshold=0.85
-    )
-    
-    result = await crawler.arun(
-        urls=[
-            "https://aws.amazon.com/ec2/pricing/",
-            "https://cloud.google.com/gpu",
-            "https://azure.microsoft.com/pricing/"
-        ],
-        optimizer=optimizer,
-        optimization_mode="minimal_extraction"
-    )
-    
-    print(f"Knowledge Coverage: {result.knowledge_coverage}")
-    print(f"Data Efficiency: {result.efficiency_ratio}")
-    print(f"Extracted Content: {result.optimal_content}")
-```
-
-### 1.3 Agentic Crawler
-An autonomous system capable of understanding complex goals and automatically planning and executing multi-step crawling operations.
-
-Key Features:
-- Autonomous goal interpretation
-- Dynamic step planning
-- Interactive navigation capabilities
-- Visual recognition and interaction
-- Automatic error recovery
-
-```python
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.agents import CrawlerAgent
-
-async with AsyncWebCrawler() as crawler:
-    agent = CrawlerAgent(crawler)
-    
-    # Automatic planning and execution
-    result = await agent.arun(
-        goal="Find research papers about quantum computing published in 2023 with more than 50 citations",
-        auto_retry=True
-    )
-    print("Generated Plan:", result.executed_steps)
-    print("Extracted Data:", result.data)
-    
-    # Using custom steps with automatic execution
-    result = await agent.arun(
-        goal="Extract conference deadlines from ML conferences",
-        custom_plan=[
-            "Navigate to conference page",
-            "Find important dates section",
-            "Extract submission deadlines",
-            "Verify dates are for 2024"
-        ]
-    )
-    
-    # Monitoring execution
-    print("Step Completion:", result.step_status)
-    print("Execution Time:", result.execution_time)
-    print("Success Rate:", result.success_rate)
-```
-
-# Section 2: Specialized Features 🛠️
-
-This section introduces specialized tools and features that enhance Crawl4AI's capabilities for specific use cases and data extraction needs.
-
-### 2.1 Automated Schema Generator
-A system that automatically generates JsonCssExtractionStrategy schemas from natural language descriptions, making structured data extraction accessible to all users.
-
-Key Features:
-- Natural language schema generation
-- Automatic pattern detection
-- Predefined schema templates
-- Chrome extension for visual schema building
-
-```python
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.schema import SchemaGenerator
-
-# Generate schema from natural language description
-generator = SchemaGenerator()
-schema = await generator.generate(
-    url="https://news-website.com",
-    description="For each news article on the page, I need the headline, publication date, and main image"
-)
-
-# Use generated schema with crawler
-async with AsyncWebCrawler() as crawler:
-    result = await crawler.arun(
-        url="https://news-website.com",
-        extraction_strategy=schema
-    )
-
-# Example of generated schema:
-"""
-{
-    "name": "News Article Extractor",
-    "baseSelector": "article.news-item",
-    "fields": [
-        {
-            "name": "headline",
-            "selector": "h2.article-title",
-            "type": "text"
-        },
-        {
-            "name": "date",
-            "selector": "span.publish-date",
-            "type": "text"
-        },
-        {
-            "name": "image",
-            "selector": "img.article-image",
-            "type": "attribute",
-            "attribute": "src"
-        }
-    ]
-}
-"""
-```
-
-### 2.2 Domain Specific Scrapers
-Specialized extraction strategies optimized for common website types and platforms, providing consistent and reliable data extraction without additional configuration.
-
-Key Features:
-- Pre-configured extractors for popular platforms
-- Academic site specialization (arXiv, NCBI)
-- E-commerce standardization
-- Documentation site handling
-
-```python
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.extractors import AcademicExtractor, EcommerceExtractor
-
-async with AsyncWebCrawler() as crawler:
-    # Academic paper extraction
-    papers = await crawler.arun(
-        url="https://arxiv.org/list/cs.AI/recent",
-        extractor="academic",  # Built-in extractor type
-        site_type="arxiv",     # Specific site optimization
-        extract_fields=[
-            "title", 
-            "authors", 
-            "abstract", 
-            "citations"
-        ]
-    )
-    
-    # E-commerce product data
-    products = await crawler.arun(
-        url="https://store.example.com/products",
-        extractor="ecommerce",
-        extract_fields=[
-            "name",
-            "price",
-            "availability",
-            "reviews"
-        ]
-    )
-```
-
-### 2.3 Web Embedding Index
-Creates and maintains a semantic search infrastructure for crawled content, enabling efficient retrieval and querying of web content through vector embeddings.
-
-Key Features:
-- Automatic embedding generation
-- Intelligent content chunking
-- Efficient vector storage and indexing
-- Semantic search capabilities
-
-```python
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.indexing import WebIndex
-
-# Initialize and build index
-index = WebIndex(model="efficient-mini")
-
-async with AsyncWebCrawler() as crawler:
-    # Crawl and index content
-    await index.build(
-        urls=["https://docs.example.com"],
-        crawler=crawler,
-        options={
-            "chunk_method": "semantic",
-            "update_policy": "incremental",
-            "embedding_batch_size": 100
-        }
-    )
-
-    # Search through indexed content
-    results = await index.search(
-        query="How to implement OAuth authentication?",
-        filters={
-            "content_type": "technical",
-            "recency": "6months"
-        },
-        top_k=5
-    )
-
-    # Get similar content
-    similar = await index.find_similar(
-        url="https://docs.example.com/auth/oauth",
-        threshold=0.85
-    )
-```
-
-Each of these specialized features builds upon Crawl4AI's core functionality while providing targeted solutions for specific use cases. They can be used independently or combined for more complex data extraction and processing needs.
-
-# Section 3: Development Tools 🔧
-
-This section covers tools designed to enhance the development experience, monitoring, and deployment of Crawl4AI applications.
-
-### 3.1 Crawl4AI Playground 🎮
-
-The Crawl4AI Playground is an interactive web-based development environment that simplifies web scraping experimentation, development, and deployment. With its intuitive interface and AI-powered assistance, users can quickly prototype, test, and deploy web scraping solutions.
-
-#### Key Features 🌟
-
-##### Visual Strategy Builder
-- Interactive point-and-click interface for building extraction strategies
-- Real-time preview of selected elements
-- Side-by-side comparison of different extraction approaches
-- Visual validation of CSS selectors and XPath queries
-
-##### AI Assistant Integration
-- Strategy recommendations based on target website analysis
-- Parameter optimization suggestions
-- Best practices guidance for specific use cases
-- Automated error detection and resolution
-- Performance optimization tips
-
-##### Real-Time Testing & Validation
-- Live preview of extraction results
-- Side-by-side comparison of multiple strategies
-- Performance metrics visualization
-- Automatic validation of extracted data
-- Error detection and debugging tools
-
-##### Project Management
-- Save and organize multiple scraping projects
-- Version control for configurations
-- Export/import project settings
-- Share configurations with team members
-- Project templates for common use cases
-
-##### Deployment Pipeline
-- One-click deployment to various environments
-- Docker container generation
-- Cloud deployment templates (AWS, GCP, Azure)
-- Scaling configuration management
-- Monitoring setup automation
-
-
-### 3.2 Performance Monitoring System
-A comprehensive monitoring solution providing real-time insights into crawler operations, resource usage, and system health through both CLI and GUI interfaces.
-
-Key Features:
-- Real-time resource tracking
-- Active crawl monitoring
-- Performance statistics
-- Customizable alerting system
-
-```python
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.monitor import CrawlMonitor
-
-# Initialize monitoring
-monitor = CrawlMonitor()
-
-# Start monitoring with CLI interface
-await monitor.start(
-    mode="cli",  # or "gui"
-    refresh_rate="1s",
-    metrics={
-        "resources": ["cpu", "memory", "network"],
-        "crawls": ["active", "queued", "completed"],
-        "performance": ["success_rate", "response_times"]
-    }
-)
-
-# Example CLI output:
-"""
-Crawl4AI Monitor (Live) - Press Q to exit
-────────────────────────────────────────
-System Usage:
- ├─ CPU: ███████░░░ 70%
- └─ Memory: ████░░░░░ 2.1GB/8GB
-
-Active Crawls:
-ID    URL                   Status    Progress
-001   docs.example.com     🟢 Active   75%
-002   api.service.com      🟡 Queue    -
-
-Metrics (Last 5min):
- ├─ Success Rate: 98%
- ├─ Avg Response: 0.6s
- └─ Pages/sec: 8.5
-"""
-```
-
-### 3.3 Cloud Integration
-Streamlined deployment tools for setting up Crawl4AI in various cloud environments, with support for scaling and monitoring.
-
-Key Features:
-- One-click deployment solutions
-- Auto-scaling configuration
-- Load balancing setup
-- Cloud-specific optimizations
-- Monitoring integration
-
-```python
-from crawl4ai import AsyncWebCrawler
-from crawl4ai.deploy import CloudDeployer
-
-# Initialize deployer
-deployer = CloudDeployer()
-
-# Deploy crawler service
-deployment = await deployer.deploy(
-    service_name="crawler-cluster",
-    platform="aws",  # or "gcp", "azure"
-    config={
-        "instance_type": "compute-optimized",
-        "auto_scaling": {
-            "min_instances": 2,
-            "max_instances": 10,
-            "scale_based_on": "cpu_usage"
-        },
-        "region": "us-east-1",
-        "monitoring": True
-    }
-)
-
-# Get deployment status and endpoints
-print(f"Service Status: {deployment.status}")
-print(f"API Endpoint: {deployment.endpoint}")
-print(f"Monitor URL: {deployment.monitor_url}")
-```
-
-These development tools work together to provide a comprehensive environment for developing, testing, monitoring, and deploying Crawl4AI applications. The Playground helps users experiment and generate optimal configurations, the Performance Monitor ensures smooth operation, and the Cloud Integration tools simplify deployment and scaling.
-
-# Section 4: Community & Growth 🌱
-
-This section outlines initiatives designed to build and support the Crawl4AI community, provide educational resources, and ensure sustainable project growth.
-
-### 4.1 Sponsorship Program
-A structured program to support ongoing development and maintenance of Crawl4AI while providing valuable benefits to sponsors.
-
-Key Features:
-- Multiple sponsorship tiers
-- Sponsor recognition system
-- Priority support for sponsors
-- Early access to new features
-- Custom feature development opportunities
-
-Program Structure (not yet finalized):
-```
-Sponsorship Tiers:
-
-🥉 Bronze Supporter
-- GitHub Sponsor badge
-- Priority issue response
-- Community Discord role
-
-🥈 Silver Supporter
-- All Bronze benefits
-- Technical support channel
-- Vote on roadmap priorities
-- Early access to beta features
-
-🥇 Gold Supporter
-- All Silver benefits
-- Custom feature requests
-- Direct developer access
-- Private support sessions
-
-💎 Diamond Partner
-- All Gold benefits
-- Custom development
-- On-demand consulting
-- Integration support
-```
-
-### 4.2 "How to Crawl" Video Series
-A comprehensive educational resource teaching users how to effectively use Crawl4AI for various web scraping and data extraction scenarios.
-
-Key Features:
-- Step-by-step tutorials
-- Real-world use cases
-- Best practices
-- Integration guides
-- Advanced feature deep-dives
-
-These community initiatives are designed to:
-- Provide comprehensive learning resources
-- Foster a supportive user community
-- Ensure sustainable project development
-- Share knowledge and best practices
-- Create opportunities for collaboration
-
-The combination of structured support through sponsorship, educational content through video series, and interactive learning through the playground creates a robust ecosystem for both new and experienced users of Crawl4AI.

SPONSORS.md

@@ -1,65 +0,0 @@
-# 💖 Sponsors & Supporters
-
-Thank you to everyone supporting Crawl4AI! Your sponsorship helps keep this project open-source and actively maintained.
-
-## 👑 Founding Sponsors
-*The first 50 sponsors who believed in our vision - permanently recognized*
-
-<!-- Founding sponsors will be listed here with special recognition -->
-🎉 **Become a Founding Sponsor!** Only [X/50] spots remaining! [Join now →](https://github.com/sponsors/unclecode)
-
----
-
-## 🏢 Data Infrastructure Partners ($2000/month)
-*These organizations are building their data sovereignty with Crawl4AI at the core*
-
-<!-- Data Infrastructure Partners will be listed here -->
-*Be the first Data Infrastructure Partner! [Join us →](https://github.com/sponsors/unclecode)*
-
----
-
-## 💼 Growing Teams ($500/month)
-*Teams scaling their data extraction with Crawl4AI*
-
-<!-- Growing Teams will be listed here -->
-*Your team could be here! [Become a sponsor →](https://github.com/sponsors/unclecode)*
-
----
-
-## 🚀 Builders ($50/month)
-*Developers and entrepreneurs building with Crawl4AI*
-
-<!-- Builders will be listed here -->
-*Join the builders! [Start sponsoring →](https://github.com/sponsors/unclecode)*
-
----
-
-## 🌱 Believers ($5/month)
-*The community supporting data democratization*
-
-<!-- Believers will be listed here -->
-*Thank you to all our community believers!*
-
----
-
-## 🤝 Want to Sponsor?
-
-Crawl4AI is the #1 trending open-source web crawler. We're building the future of data extraction - where organizations own their data pipelines instead of relying on rate-limited APIs.
-
-### Available Sponsorship Tiers:
-- **🌱 Believer** ($5/mo) - Support the movement
-- **🚀 Builder** ($50/mo) - Priority support & early access
-- **💼 Growing Team** ($500/mo) - Bi-weekly syncs & optimization
-- **🏢 Data Infrastructure Partner** ($2000/mo) - Full partnership & dedicated support
-
-[View all tiers and benefits →](https://github.com/sponsors/unclecode)
-
-### Enterprise & Custom Partnerships
-
-Building data extraction at scale? Need dedicated support or infrastructure? Let's talk about a custom partnership.
-
-📧 Contact: [hello@crawl4ai.com](mailto:hello@crawl4ai.com) | 📅 [Schedule a call](https://calendar.app.google/rEpvi2UBgUQjWHfJ9)
-
----
-
-*This list is updated regularly. Sponsors at $50+ tiers can submit their logos via [hello@crawl4ai.com](mailto:hello@crawl4ai.com)*

cliff.toml

@@ -1,24 +0,0 @@
-[changelog]
-# Template format
-header = """
-# Changelog\n
-All notable changes to this project will be documented in this file.\n
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n
-"""
-
-# Organize commits by type
-[git]
-conventional_commits = true
-filter_unconventional = true
-commit_parsers = [
-    { message = "^feat", group = "Added"},
-    { message = "^fix", group = "Fixed"},
-    { message = "^doc", group = "Documentation"},
-    { message = "^perf", group = "Performance"},
-    { message = "^refactor", group = "Changed"},
-    { message = "^style", group = "Changed"},
-    { message = "^test", group = "Testing"},
-    { message = "^chore\\(release\\): prepare for", skip = true},
-    { message = "^chore", group = "Miscellaneous Tasks"},
-]

crawl4ai/__init__.py

@@ -1,226 +1,30 @@
 # __init__.py
-import warnings
 
-from .async_webcrawler import AsyncWebCrawler, CacheMode
-# MODIFIED: Add SeedingConfig and VirtualScrollConfig here
-from .async_configs import BrowserConfig, CrawlerRunConfig, HTTPCrawlerConfig, LLMConfig, ProxyConfig, GeolocationConfig, SeedingConfig, VirtualScrollConfig, LinkPreviewConfig, MatchMode
+from .async_webcrawler import AsyncWebCrawler
+from .models import CrawlResult
 
-from .content_scraping_strategy import (
-    ContentScrapingStrategy,
-    LXMLWebScrapingStrategy,
-    WebScrapingStrategy,  # Backward compatibility alias
-)
-from .async_logger import (
-    AsyncLoggerBase,
-    AsyncLogger,
-)
-from .proxy_strategy import (
-    ProxyRotationStrategy,
-    RoundRobinProxyStrategy,
-)
-from .extraction_strategy import (
-    ExtractionStrategy,
-    LLMExtractionStrategy,
-    CosineStrategy,
-    JsonCssExtractionStrategy,
-    JsonXPathExtractionStrategy,
-    JsonLxmlExtractionStrategy,
-    RegexExtractionStrategy
-)
-from .chunking_strategy import ChunkingStrategy, RegexChunking
-from .markdown_generation_strategy import DefaultMarkdownGenerator
-from .table_extraction import (
-    TableExtractionStrategy,
-    DefaultTableExtraction,
-    NoTableExtraction,
-    LLMTableExtraction,
-)
-from .content_filter_strategy import (
-    PruningContentFilter,
-    BM25ContentFilter,
-    LLMContentFilter,
-    RelevantContentFilter,
-)
-from .models import CrawlResult, MarkdownGenerationResult, DisplayMode
-from .components.crawler_monitor import CrawlerMonitor
-from .link_preview import LinkPreview
-from .async_dispatcher import (
-    MemoryAdaptiveDispatcher,
-    SemaphoreDispatcher,
-    RateLimiter,
-    BaseDispatcher,
-)
-from .docker_client import Crawl4aiDockerClient
-from .hub import CrawlerHub
-from .browser_profiler import BrowserProfiler
-from .deep_crawling import (
-    DeepCrawlStrategy,
-    BFSDeepCrawlStrategy,
-    FilterChain,
-    URLPatternFilter,
-    DomainFilter,
-    ContentTypeFilter,
-    URLFilter,
-    FilterStats,
-    SEOFilter,
-    KeywordRelevanceScorer,
-    URLScorer,
-    CompositeScorer,
-    DomainAuthorityScorer,
-    FreshnessScorer,
-    PathDepthScorer,
-    BestFirstCrawlingStrategy,
-    DFSDeepCrawlStrategy,
-    DeepCrawlDecorator,
-)
-# NEW: Import AsyncUrlSeeder
-from .async_url_seeder import AsyncUrlSeeder
-# Adaptive Crawler
-from .adaptive_crawler import (
-    AdaptiveCrawler,
-    AdaptiveConfig,
-    CrawlState,
-    CrawlStrategy,
-    StatisticalStrategy
-)
-
-# C4A Script Language Support
-from .script import (
-    compile as c4a_compile,
-    validate as c4a_validate,
-    compile_file as c4a_compile_file,
-    CompilationResult,
-    ValidationResult,
-    ErrorDetail
-)
-
-# Browser Adapters
-from .browser_adapter import (
-    BrowserAdapter,
-    PlaywrightAdapter,
-    UndetectedAdapter
-)
-
-from .utils import (
-    start_colab_display_server,
-    setup_colab_environment
-)
+__version__ = "0.3.6"
 
 __all__ = [
-    "AsyncLoggerBase",
-    "AsyncLogger",
     "AsyncWebCrawler",
-    "BrowserProfiler",
-    "LLMConfig",
-    "GeolocationConfig",
-    # NEW: Add SeedingConfig and VirtualScrollConfig
-    "SeedingConfig",
-    "VirtualScrollConfig",
-    # NEW: Add AsyncUrlSeeder
-    "AsyncUrlSeeder",
-    # Adaptive Crawler
-    "AdaptiveCrawler",
-    "AdaptiveConfig", 
-    "CrawlState",
-    "CrawlStrategy",
-    "StatisticalStrategy",
-    "DeepCrawlStrategy",
-    "BFSDeepCrawlStrategy",
-    "BestFirstCrawlingStrategy",
-    "DFSDeepCrawlStrategy",
-    "FilterChain",
-    "URLPatternFilter",
-    "ContentTypeFilter",
-    "DomainFilter",
-    "FilterStats",
-    "URLFilter",
-    "SEOFilter",
-    "KeywordRelevanceScorer",
-    "URLScorer",
-    "CompositeScorer",
-    "DomainAuthorityScorer",
-    "FreshnessScorer",
-    "PathDepthScorer",
-    "DeepCrawlDecorator",
     "CrawlResult",
-    "CrawlerHub",
-    "CacheMode",
-    "MatchMode",
-    "ContentScrapingStrategy",
-    "WebScrapingStrategy",
-    "LXMLWebScrapingStrategy",
-    "BrowserConfig",
-    "CrawlerRunConfig",
-    "HTTPCrawlerConfig",
-    "ExtractionStrategy",
-    "LLMExtractionStrategy",
-    "CosineStrategy",
-    "JsonCssExtractionStrategy",
-    "JsonXPathExtractionStrategy",
-    "JsonLxmlExtractionStrategy",
-    "RegexExtractionStrategy",
-    "ChunkingStrategy",
-    "RegexChunking",
-    "DefaultMarkdownGenerator",
-    "TableExtractionStrategy",
-    "DefaultTableExtraction",
-    "NoTableExtraction",
-    "RelevantContentFilter",
-    "PruningContentFilter",
-    "BM25ContentFilter",
-    "LLMContentFilter",
-    "BaseDispatcher",
-    "MemoryAdaptiveDispatcher",
-    "SemaphoreDispatcher",
-    "RateLimiter",
-    "CrawlerMonitor",
-    "LinkPreview",
-    "DisplayMode",
-    "MarkdownGenerationResult",
-    "Crawl4aiDockerClient",
-    "ProxyRotationStrategy",
-    "RoundRobinProxyStrategy",
-    "ProxyConfig",
-    "start_colab_display_server",
-    "setup_colab_environment",
-    # C4A Script additions
-    "c4a_compile",
-    "c4a_validate", 
-    "c4a_compile_file",
-    "CompilationResult",
-    "ValidationResult",
-    "ErrorDetail",
-    # Browser Adapters
-    "BrowserAdapter",
-    "PlaywrightAdapter", 
-    "UndetectedAdapter",
-    "LinkPreviewConfig"
 ]
 
+def is_sync_version_installed():
+    try:
+        import selenium
+        return True
+    except ImportError:
+        return False
 
-# def is_sync_version_installed():
-#     try:
-#         import selenium # noqa
-
-#         return True
-#     except ImportError:
-#         return False
-
-
-# if is_sync_version_installed():
-#     try:
-#         from .web_crawler import WebCrawler
-
-#         __all__.append("WebCrawler")
-#     except ImportError:
-#         print(
-#             "Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies."
-#         )
-# else:
-#     WebCrawler = None
-#     # import warnings
-#     # print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.")
-
-# Disable all Pydantic warnings
-warnings.filterwarnings("ignore", module="pydantic")
-# pydantic_warnings.filter_warnings()
+if is_sync_version_installed():
+    try:
+        from .web_crawler import WebCrawler
+        __all__.append("WebCrawler")
+    except ImportError:
+        import warnings
+        print("Warning: Failed to import WebCrawler even though selenium is installed. This might be due to other missing dependencies.")
+else:
+    WebCrawler = None
+    import warnings
+    print("Warning: Synchronous WebCrawler is not available. Install crawl4ai[sync] for synchronous support. However, please note that the synchronous version will be deprecated soon.")

crawl4ai/__version__.py

@@ -1,8 +0,0 @@
-# crawl4ai/__version__.py
-
-# This is the version that will be used for stable releases
-__version__ = "0.7.4"
-
-# For nightly builds, this gets set during build process
-__nightly_version__ = None
-

crawl4ai/agent/FIXED.md

@@ -1,73 +0,0 @@
-# ✅ FIXED: Chat Mode Now Fully Functional!
-
-## Issues Resolved:
-
-### Issue 1: Agent wasn't responding with text ❌ → ✅ FIXED
-**Problem:** After tool execution, no response text was shown
-**Root Cause:** Not extracting text from `message_output_item.raw_item.content[].text`
-**Fix:** Added proper extraction from content blocks
-
-### Issue 2: Chat didn't continue after first turn ❌ → ✅ FIXED
-**Problem:** Chat appeared stuck, no response to follow-up questions
-**Root Cause:** Same as Issue 1 - responses weren't being displayed
-**Fix:** Chat loop was always working, just needed to show the responses
-
----
-
-## Working Example:
-
-```
-You: Crawl example.com and tell me the title
-
-Agent: thinking...
-
-🔧 Calling: quick_crawl
-  (url=https://example.com, output_format=markdown)
-  ✓ completed
-
-Agent: The title of the page at example.com is:
-
-Example Domain
-
-Let me know if you need more information from this site!
-
-Tools used: quick_crawl
-
-You: So what is it?
-
-Agent: thinking...
-
-Agent: The title is "Example Domain" - this is a standard placeholder...
-```
-
----
-
-## Test It Now:
-
-```bash
-export OPENAI_API_KEY="sk-..."
-python -m crawl4ai.agent.agent_crawl --chat
-```
-
-Then try:
-```
-Crawl example.com and tell me the title
-What else can you tell me about it?
-Start a session called 'test' and navigate to example.org
-Extract the markdown
-Close the session
-/exit
-```
-
----
-
-## What Works:
-
-✅ Full streaming visibility
-✅ Tool calls shown with arguments
-✅ Agent responses shown
-✅ Multi-turn conversations
-✅ Session management
-✅ All 7 tools working
-
-**Everything is working perfectly now!** 🎉

crawl4ai/agent/MIGRATION_SUMMARY.md

@@ -1,141 +0,0 @@
-# Crawl4AI Agent - Claude SDK → OpenAI SDK Migration
-
-**Status:** ✅ Complete
-**Date:** 2025-10-17
-
-## What Changed
-
-### Files Created/Rewritten:
-1. ✅ `crawl_tools.py` - Converted from Claude SDK `@tool` to OpenAI SDK `@function_tool`
-2. ✅ `crawl_prompts.py` - Cleaned up prompt (removed Claude-specific references)
-3. ✅ `agent_crawl.py` - Complete rewrite using OpenAI `Agent` + `Runner`
-4. ✅ `chat_mode.py` - Rewrit with **streaming visibility** and real-time status updates
-
-### Files Kept (No Changes):
-- ✅ `browser_manager.py` - Singleton pattern is SDK-agnostic
-- ✅ `terminal_ui.py` - Minor updates (added /browser command)
-
-### Files Backed Up:
-- `agent_crawl.py.old` - Original Claude SDK version
-- `chat_mode.py.old` - Original Claude SDK version
-
-## Key Improvements
-
-### 1. **No CLI Dependency**
-- ❌ OLD: Spawned `claude` CLI subprocess
-- ✅ NEW: Direct OpenAI API calls
-
-### 2. **Cleaner Tool API**
-```python
-# OLD (Claude SDK)
-@tool("quick_crawl", "Description", {"url": str, ...})
-async def quick_crawl(args: Dict[str, Any]) -> Dict[str, Any]:
-    return {"content": [{"type": "text", "text": json.dumps(...)}]}
-
-# NEW (OpenAI SDK)
-@function_tool
-async def quick_crawl(url: str, output_format: str = "markdown", ...) -> str:
-    return json.dumps(...)  # Direct return
-```
-
-### 3. **Simpler Execution**
-```python
-# OLD (Claude SDK)
-async with ClaudeSDKClient(options) as client:
-    await client.query(message_generator())
-    async for message in client.receive_messages():
-        # Complex message handling...
-
-# NEW (OpenAI SDK)
-result = await Runner.run(agent, input=prompt, context=None)
-print(result.final_output)
-```
-
-### 4. **Streaming Chat with Visibility** (MAIN FEATURE!)
-
-The new chat mode shows:
-- ✅ **"thinking..."** indicator when agent starts
-- ✅ **Tool calls** with parameters: `🔧 Calling: quick_crawl (url=example.com)`
-- ✅ **Tool completion**: `✓ completed`
-- ✅ **Real-time text streaming** character-by-character
-- ✅ **Summary** after response: Tools used, token count
-- ✅ **Clear status** at every step
-
-**Example output:**
-```
-You: Crawl example.com and extract the title
-
-Agent: thinking...
-
-🔧 Calling: quick_crawl
-  (url=https://example.com, output_format=markdown)
-  ✓ completed
-
-Agent: I've successfully crawled example.com. The title is "Example Domain"...
-
-Tools used: quick_crawl
-Tokens: input=45, output=23
-```
-
-## Installation
-
-```bash
-# Install OpenAI Agents SDK
-pip install git+https://github.com/openai/openai-agents-python.git
-
-# Set API key
-export OPENAI_API_KEY="sk-..."
-```
-
-## Usage
-
-### Chat Mode (Recommended):
-```bash
-python -m crawl4ai.agent.agent_crawl --chat
-```
-
-### Single-Shot Mode:
-```bash
-python -m crawl4ai.agent.agent_crawl "Crawl example.com"
-```
-
-### Commands in Chat:
-- `/exit` - Exit chat
-- `/clear` - Clear screen
-- `/help` - Show help
-- `/browser` - Show browser status
-
-## Testing
-
-Tests need to be updated (not done yet):
-- ❌ `test_chat.py` - Update for OpenAI SDK
-- ❌ `test_tools.py` - Update execution model
-- ❌ `test_scenarios.py` - Update multi-turn tests
-- ❌ `run_all_tests.py` - Update imports
-
-## Migration Benefits
-
-| Metric | Claude SDK | OpenAI SDK | Improvement |
-|--------|------------|------------|-------------|
-| **Startup Time** | ~2s (CLI spawn) | ~0.1s | **20x faster** |
-| **Dependencies** | Node.js + CLI | Python only | **Simpler** |
-| **Session Isolation** | Shared `~/.claude/` | Isolated | **Cleaner** |
-| **Tool API** | Dict-based | Type-safe | **Better DX** |
-| **Visibility** | Minimal | Full streaming | **Much better** |
-| **Production Ready** | No (CLI dep) | Yes | **Production** |
-
-## Known Issues
-
-- OpenAI SDK upgraded to 2.4.0, conflicts with:
-  - `instructor` (requires <2.0.0)
-  - `pandasai` (requires <2)
-  - `shell-gpt` (requires <2.0.0)
-
-  These are acceptable conflicts if you're not using those packages.
-
-## Next Steps
-
-1. Test the new chat mode thoroughly
-2. Update test files
-3. Update documentation
-4. Consider adding more streaming events (progress bars, etc.)

crawl4ai/agent/READY.md

@@ -1,172 +0,0 @@
-# ✅ Crawl4AI Agent - OpenAI SDK Migration Complete
-
-## Status: READY TO USE
-
-All migration completed and tested successfully!
-
----
-
-## What's New
-
-### 🚀 Key Improvements:
-
-1. **No CLI Dependency** - Direct OpenAI API calls (20x faster startup)
-2. **Full Visibility** - See every tool call, argument, and status in real-time
-3. **Cleaner Code** - 50% less code, type-safe tools
-4. **Better UX** - Streaming responses with clear status indicators
-
----
-
-## Usage
-
-### Chat Mode (Recommended):
-```bash
-export OPENAI_API_KEY="sk-..."
-python -m crawl4ai.agent.agent_crawl --chat
-```
-
-**What you'll see:**
-```
-🕷️  Crawl4AI Agent - Chat Mode
-Powered by OpenAI Agents SDK
-
-You: Crawl example.com and get the title
-
-Agent: thinking...
-
-🔧 Calling: quick_crawl
-  (url=https://example.com, output_format=markdown)
-  ✓ completed
-
-Agent: The title of example.com is "Example Domain"
-
-Tools used: quick_crawl
-```
-
-### Single-Shot Mode:
-```bash
-python -m crawl4ai.agent.agent_crawl "Get title from example.com"
-```
-
-### Commands in Chat:
-- `/exit` - Exit chat
-- `/clear` - Clear screen
-- `/help` - Show help
-- `/browser` - Browser status
-
----
-
-## Files Changed
-
-### ✅ Created/Rewritten:
-- `crawl_tools.py` - 7 tools with `@function_tool` decorator
-- `crawl_prompts.py` - Clean system prompt
-- `agent_crawl.py` - Simple Agent + Runner
-- `chat_mode.py` - Streaming chat with full visibility
-- `__init__.py` - Updated exports
-
-### ✅ Updated:
-- `terminal_ui.py` - Added /browser command
-
-### ✅ Unchanged:
-- `browser_manager.py` - Works perfectly as-is
-
-### ❌ Removed:
-- `c4ai_tools.py` (old Claude SDK tools)
-- `c4ai_prompts.py` (old prompts)
-- All `.old` backup files
-
----
-
-## Tests Performed
-
-✅ **Import Tests** - All modules import correctly
-✅ **Agent Creation** - Agent created with 7 tools
-✅ **Single-Shot Mode** - Successfully crawled example.com
-✅ **Chat Mode Streaming** - Full visibility working:
-   - Shows "thinking..." indicator
-   - Shows tool calls: `🔧 Calling: quick_crawl`
-   - Shows arguments: `(url=https://example.com, output_format=markdown)`
-   - Shows completion: `✓ completed`
-   - Shows summary: `Tools used: quick_crawl`
-
----
-
-## Chat Mode Features (YOUR MAIN REQUEST!)
-
-### Real-Time Visibility:
-
-1. **Thinking Indicator**
-   ```
-   Agent: thinking...
-   ```
-
-2. **Tool Calls with Arguments**
-   ```
-   🔧 Calling: quick_crawl
-     (url=https://example.com, output_format=markdown)
-   ```
-
-3. **Tool Completion**
-   ```
-     ✓ completed
-   ```
-
-4. **Agent Response (Streaming)**
-   ```
-   Agent: The title is "Example Domain"...
-   ```
-
-5. **Summary**
-   ```
-   Tools used: quick_crawl
-   ```
-
-You now have **complete observability** - you'll see exactly what the agent is doing at every step!
-
----
-
-## Migration Stats
-
-| Metric | Before (Claude SDK) | After (OpenAI SDK) |
-|--------|---------------------|-------------------|
-| Lines of code | ~400 | ~200 |
-| Startup time | 2s | 0.1s |
-| Dependencies | Node.js + CLI | Python only |
-| Visibility | Minimal | Full streaming |
-| Tool API | Dict-based | Type-safe |
-| Production ready | No | Yes |
-
----
-
-## Known Issues
-
-None! Everything tested and working.
-
----
-
-## Next Steps (Optional)
-
-1. Update test files (`test_chat.py`, `test_tools.py`, `test_scenarios.py`)
-2. Add more streaming events (progress bars, etc.)
-3. Add session persistence
-4. Add conversation history
-
----
-
-## Try It Now!
-
-```bash
-cd /Users/unclecode/devs/crawl4ai
-export OPENAI_API_KEY="sk-..."
-python -m crawl4ai.agent.agent_crawl --chat
-```
-
-Then try:
-```
-Crawl example.com and extract the title
-Start session 'test', navigate to example.org, and extract the markdown
-Close the session
-```
-
-Enjoy your new agent with **full visibility**! 🎉

crawl4ai/agent/TECH_SPEC.md

@@ -1,429 +0,0 @@
-# Crawl4AI Agent Technical Specification
-*AI-to-AI Knowledge Transfer Document*
-
-## Context Documents
-**MUST READ FIRST:**
-1. `/Users/unclecode/devs/crawl4ai/tmp/CRAWL4AI_SDK.md` - Crawl4AI complete API reference
-2. `/Users/unclecode/devs/crawl4ai/tmp/cc_stream.md` - Claude SDK streaming input mode
-3. `/Users/unclecode/devs/crawl4ai/tmp/CC_PYTHON_SDK.md` - Claude Code Python SDK complete reference
-
-## Architecture Overview
-
-**Core Principle:** Singleton browser instance + streaming chat mode + MCP tools
-
-```
-┌─────────────────────────────────────────────────────────────┐
-│                    Agent Entry Point                         │
-│         agent_crawl.py (CLI: --chat | single-shot)          │
-└─────────────────────────────────────────────────────────────┘
-                            │
-        ┌───────────────────┼───────────────────┐
-        │                   │                   │
-   [Chat Mode]         [Single-shot]    [Browser Manager]
-        │                   │                   │
-        ▼                   ▼                   ▼
-  ChatMode.run()    CrawlAgent.run()   BrowserManager
-  - Streaming        - One prompt          (Singleton)
-  - Interactive      - Exit after           │
-  - Commands         - Uses same            ▼
-        │              browser         AsyncWebCrawler
-        │                   │             (persistent)
-        └───────────────────┴────────────────┘
-                            │
-                    ┌───────┴────────┐
-                    │                │
-              MCP Tools        Claude SDK
-            (Crawl4AI)        (Built-in)
-                    │                │
-        ┌───────────┴────┐    ┌──────┴──────┐
-        │                │    │             │
-   quick_crawl    session    Read        Edit
-   navigate       tools      Write       Glob
-   extract_data              Bash        Grep
-   execute_js
-   screenshot
-   close_session
-```
-
-## File Structure
-
-```
-crawl4ai/agent/
-├── __init__.py                 # Module exports
-├── agent_crawl.py              # Main CLI entry (190 lines)
-│   ├── SessionStorage          # JSONL logging to ~/.crawl4ai/agents/projects/
-│   ├── CrawlAgent             # Single-shot wrapper
-│   └── main()                 # CLI parser (--chat flag)
-│
-├── browser_manager.py          # Singleton pattern (70 lines)
-│   └── BrowserManager         # Class methods only, no instances
-│       ├── get_browser()      # Returns singleton AsyncWebCrawler
-│       ├── reconfigure_browser()
-│       ├── close_browser()
-│       └── is_browser_active()
-│
-├── c4ai_tools.py               # 7 MCP tools (310 lines)
-│   ├── @tool decorators       # Claude SDK decorator
-│   ├── CRAWLER_SESSIONS       # Dict[str, AsyncWebCrawler] for named sessions
-│   ├── CRAWLER_SESSION_URLS   # Dict[str, str] track current URL per session
-│   └── CRAWL_TOOLS            # List of tool functions
-│
-├── c4ai_prompts.py             # System prompt (130 lines)
-│   └── SYSTEM_PROMPT          # Agent behavior definition
-│
-├── terminal_ui.py              # Rich-based UI (120 lines)
-│   └── TerminalUI             # Console rendering
-│       ├── show_header()
-│       ├── print_markdown()
-│       ├── print_code()
-│       └── with_spinner()
-│
-├── chat_mode.py                # Streaming chat (160 lines)
-│   └── ChatMode
-│       ├── message_generator() # AsyncGenerator per cc_stream.md
-│       ├── _handle_command()   # /exit /clear /help /browser
-│       └── run()              # Main chat loop
-│
-├── test_tools.py               # Direct tool tests (130 lines)
-├── test_chat.py                # Component tests (90 lines)
-└── test_scenarios.py           # Multi-turn scenarios (500 lines)
-    ├── SIMPLE_SCENARIOS
-    ├── MEDIUM_SCENARIOS
-    ├── COMPLEX_SCENARIOS
-    └── ScenarioRunner
-```
-
-## Critical Implementation Details
-
-### 1. Browser Singleton Pattern
-
-**Key:** ONE browser instance for ENTIRE agent session
-
-```python
-# browser_manager.py
-class BrowserManager:
-    _crawler: Optional[AsyncWebCrawler] = None  # Singleton
-    _config: Optional[BrowserConfig] = None
-
-    @classmethod
-    async def get_browser(cls, config=None) -> AsyncWebCrawler:
-        if cls._crawler is None:
-            cls._crawler = AsyncWebCrawler(config or BrowserConfig())
-            await cls._crawler.start()  # Manual lifecycle
-        return cls._crawler
-```
-
-**Behavior:**
-- First call: creates browser with `config` (or default)
-- Subsequent calls: returns same instance, **ignores config param**
-- To change config: `reconfigure_browser(new_config)` (closes old, creates new)
-- Tools use: `crawler = await BrowserManager.get_browser()`
-- No `async with` context manager - manual `start()` / `close()`
-
-### 2. Tool Architecture
-
-**Two types of browser usage:**
-
-**A) Quick operations** (quick_crawl):
-```python
-@tool("quick_crawl", ...)
-async def quick_crawl(args):
-    crawler = await BrowserManager.get_browser()  # Singleton
-    result = await crawler.arun(url=args["url"], config=run_config)
-    # No close - browser stays alive
-```
-
-**B) Named sessions** (start_session, navigate, extract_data, etc.):
-```python
-CRAWLER_SESSIONS: Dict[str, AsyncWebCrawler] = {}  # Named refs
-CRAWLER_SESSION_URLS: Dict[str, str] = {}  # Track current URL
-
-@tool("start_session", ...)
-async def start_session(args):
-    crawler = await BrowserManager.get_browser()
-    CRAWLER_SESSIONS[args["session_id"]] = crawler  # Store ref
-
-@tool("navigate", ...)
-async def navigate(args):
-    crawler = CRAWLER_SESSIONS[args["session_id"]]
-    result = await crawler.arun(url=args["url"], ...)
-    CRAWLER_SESSION_URLS[args["session_id"]] = result.url  # Track URL
-
-@tool("extract_data", ...)
-async def extract_data(args):
-    crawler = CRAWLER_SESSIONS[args["session_id"]]
-    current_url = CRAWLER_SESSION_URLS[args["session_id"]]  # Must have URL
-    result = await crawler.arun(url=current_url, ...)  # Re-crawl current page
-
-@tool("close_session", ...)
-async def close_session(args):
-    CRAWLER_SESSIONS.pop(args["session_id"])  # Remove ref
-    CRAWLER_SESSION_URLS.pop(args["session_id"], None)
-    # Browser stays alive (singleton)
-```
-
-**Important:** Named sessions are just **references** to singleton browser. Multiple sessions = same browser instance.
-
-### 3. Markdown Handling (CRITICAL BUG FIX)
-
-**OLD (WRONG):**
-```python
-result.markdown_v2.raw_markdown  # DEPRECATED
-```
-
-**NEW (CORRECT):**
-```python
-# result.markdown can be:
-# - str (simple mode)
-# - MarkdownGenerationResult object (with filters)
-
-if isinstance(result.markdown, str):
-    markdown_content = result.markdown
-elif hasattr(result.markdown, 'raw_markdown'):
-    markdown_content = result.markdown.raw_markdown
-```
-
-Reference: `CRAWL4AI_SDK.md` line 614 - `markdown_v2` deprecated, use `markdown`
-
-### 4. Chat Mode Streaming Input
-
-**Per cc_stream.md:** Use message generator pattern
-
-```python
-# chat_mode.py
-async def message_generator(self) -> AsyncGenerator[Dict[str, Any], None]:
-    while not self._exit_requested:
-        user_input = await asyncio.to_thread(self.ui.get_user_input)
-
-        if user_input.startswith('/'):
-            await self._handle_command(user_input)
-            continue
-
-        # Yield in streaming input format
-        yield {
-            "type": "user",
-            "message": {
-                "role": "user",
-                "content": user_input
-            }
-        }
-
-async def run(self):
-    async with ClaudeSDKClient(options=self.options) as client:
-        await client.query(self.message_generator())  # Pass generator
-
-        async for message in client.receive_messages():
-            # Process streaming responses
-```
-
-**Key:** Generator keeps yielding user inputs, SDK streams responses back.
-
-### 5. Claude SDK Integration
-
-**Setup:**
-```python
-from claude_agent_sdk import tool, create_sdk_mcp_server, ClaudeSDKClient, ClaudeAgentOptions
-
-# 1. Define tools with @tool decorator
-@tool("quick_crawl", "description", {"url": str, "output_format": str})
-async def quick_crawl(args: Dict[str, Any]) -> Dict[str, Any]:
-    return {"content": [{"type": "text", "text": json.dumps(result)}]}
-
-# 2. Create MCP server
-crawler_server = create_sdk_mcp_server(
-    name="crawl4ai",
-    version="1.0.0",
-    tools=[quick_crawl, start_session, ...]  # List of @tool functions
-)
-
-# 3. Configure options
-options = ClaudeAgentOptions(
-    mcp_servers={"crawler": crawler_server},
-    allowed_tools=[
-        "mcp__crawler__quick_crawl",  # Format: mcp__{server}__{tool}
-        "mcp__crawler__start_session",
-        # Built-in tools:
-        "Read", "Write", "Edit", "Glob", "Grep", "Bash", "NotebookEdit"
-    ],
-    system_prompt=SYSTEM_PROMPT,
-    permission_mode="acceptEdits"
-)
-
-# 4. Use client
-async with ClaudeSDKClient(options=options) as client:
-    await client.query(prompt_or_generator)
-    async for message in client.receive_messages():
-        # Process AssistantMessage, ResultMessage, etc.
-```
-
-**Tool response format:**
-```python
-return {
-    "content": [{
-        "type": "text",
-        "text": json.dumps({"success": True, "data": "..."})
-    }]
-}
-```
-
-## Operating Modes
-
-### Single-Shot Mode
-```bash
-python -m crawl4ai.agent.agent_crawl "Crawl example.com"
-```
-- One prompt → execute → exit
-- Uses singleton browser
-- No cleanup of browser (process exit handles it)
-
-### Chat Mode
-```bash
-python -m crawl4ai.agent.agent_crawl --chat
-```
-- Interactive loop with streaming I/O
-- Commands: `/exit` `/clear` `/help` `/browser`
-- Browser persists across all turns
-- Cleanup on exit: `BrowserManager.close_browser()`
-
-## Testing Architecture
-
-**3 test levels:**
-
-1. **Component tests** (`test_chat.py`): Non-interactive, tests individual classes
-2. **Tool tests** (`test_tools.py`): Direct AsyncWebCrawler calls, validates Crawl4AI integration
-3. **Scenario tests** (`test_scenarios.py`): Automated multi-turn conversations
-   - Injects messages programmatically
-   - Validates tool calls, keywords, files created
-   - Categories: SIMPLE (2), MEDIUM (3), COMPLEX (4)
-
-## Dependencies
-
-```python
-# External
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai.extraction_strategy import LLMExtractionStrategy
-from claude_agent_sdk import (
-    tool, create_sdk_mcp_server, ClaudeSDKClient, ClaudeAgentOptions,
-    AssistantMessage, TextBlock, ResultMessage, ToolUseBlock
-)
-from rich.console import Console  # Already installed
-from rich.markdown import Markdown
-from rich.syntax import Syntax
-
-# Stdlib
-import asyncio, json, uuid, argparse
-from pathlib import Path
-from typing import Optional, Dict, Any, AsyncGenerator
-```
-
-## Common Pitfalls
-
-1. **DON'T** use `async with AsyncWebCrawler()` - breaks singleton pattern
-2. **DON'T** use `result.markdown_v2` - deprecated field
-3. **DON'T** call `crawler.arun()` without URL in session tools - needs current_url
-4. **DON'T** close browser in tools - managed by BrowserManager
-5. **DON'T** use `break` in message iteration - causes asyncio issues
-6. **DO** track session URLs in `CRAWLER_SESSION_URLS` for session tools
-7. **DO** handle both `str` and `MarkdownGenerationResult` for `result.markdown`
-8. **DO** use manual lifecycle `await crawler.start()` / `await crawler.close()`
-
-## Session Storage
-
-**Location:** `~/.crawl4ai/agents/projects/{sanitized_cwd}/{uuid}.jsonl`
-
-**Format:** JSONL with events:
-```json
-{"timestamp": "...", "event": "session_start", "data": {...}}
-{"timestamp": "...", "event": "user_message", "data": {"text": "..."}}
-{"timestamp": "...", "event": "assistant_message", "data": {"turn": 1, "text": "..."}}
-{"timestamp": "...", "event": "session_end", "data": {"duration_ms": 1000, ...}}
-```
-
-## CLI Options
-
-```
---chat                  Interactive chat mode
---model MODEL          Claude model override
---permission-mode MODE  acceptEdits|bypassPermissions|default|plan
---add-dir DIR [DIR...] Additional accessible directories
---system-prompt TEXT   Custom system prompt
---session-id UUID      Resume/specify session
---debug                Full tracebacks
-```
-
-## Performance Characteristics
-
-- **Browser startup:** ~2-4s (once per session)
-- **Quick crawl:** ~1-2s (reuses browser)
-- **Session operations:** ~1-2s (same browser)
-- **Chat latency:** Real-time streaming, no buffering
-- **Memory:** One browser instance regardless of operations
-
-## Extension Points
-
-1. **New tools:** Add `@tool` function → add to `CRAWL_TOOLS` → add to `allowed_tools`
-2. **New commands:** Add handler in `ChatMode._handle_command()`
-3. **Custom UI:** Replace `TerminalUI` with different renderer
-4. **Persistent sessions:** Serialize browser cookies/state to disk in `BrowserManager`
-5. **Multi-browser:** Modify `BrowserManager` to support multiple configs (not recommended)
-
-## Next Steps: Testing & Evaluation Pipeline
-
-### Phase 1: Automated Testing (CURRENT)
-**Objective:** Verify codebase correctness, not agent quality
-
-**Test Execution:**
-```bash
-# 1. Component tests (fast, non-interactive)
-python crawl4ai/agent/test_chat.py
-# Expected: All components instantiate correctly
-
-# 2. Tool integration tests (medium, requires browser)
-python crawl4ai/agent/test_tools.py
-# Expected: All 7 tools work with Crawl4AI
-
-# 3. Multi-turn scenario tests (slow, comprehensive)
-python crawl4ai/agent/test_scenarios.py
-# Expected: 9 scenarios pass (2 simple, 3 medium, 4 complex)
-# Output: test_agent_output/test_results.json
-```
-
-**Success Criteria:**
-- All component tests pass
-- All tool tests pass
-- ≥80% scenario tests pass (7/9)
-- No crashes, exceptions, or hangs
-- Browser cleanup verified
-
-**Automated Pipeline:**
-```bash
-# Run all tests in sequence, exit on first failure
-cd /Users/unclecode/devs/crawl4ai
-python crawl4ai/agent/test_chat.py && \
-python crawl4ai/agent/test_tools.py && \
-python crawl4ai/agent/test_scenarios.py
-echo "Exit code: $?"  # 0 = all passed
-```
-
-### Phase 2: Evaluation (NEXT)
-**Objective:** Measure agent performance quality
-
-**Metrics to define:**
-- Task completion rate
-- Tool selection accuracy
-- Context retention across turns
-- Planning effectiveness
-- Error recovery capability
-
-**Eval framework needed:**
-- Expand scenario tests with quality scoring
-- Add ground truth comparisons
-- Measure token efficiency
-- Track reasoning quality
-
-**Not in scope yet** - wait for Phase 1 completion
-
----
-**Last Updated:** 2025-01-17
-**Version:** 1.0.0
-**Status:** Testing Phase - Ready for automated test runs

crawl4ai/agent/__init__.py

@@ -1,16 +0,0 @@
-# __init__.py
-"""Crawl4AI Agent - Browser automation agent powered by OpenAI Agents SDK."""
-
-# Import only the components needed for library usage
-# Don't import agent_crawl here to avoid warning when running with python -m
-from .crawl_tools import CRAWL_TOOLS
-from .crawl_prompts import SYSTEM_PROMPT
-from .browser_manager import BrowserManager
-from .terminal_ui import TerminalUI
-
-__all__ = [
-    "CRAWL_TOOLS",
-    "SYSTEM_PROMPT",
-    "BrowserManager",
-    "TerminalUI",
-]

crawl4ai/agent/agent-cc-sdk.md

@@ -1,593 +0,0 @@
-```python
-# c4ai_tools.py
-"""Crawl4AI tools for Claude Code SDK agent."""
-
-import json
-import asyncio
-from typing import Any, Dict
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai.extraction_strategy import LLMExtractionStrategy
-from claude_agent_sdk import tool
-
-# Global session storage
-CRAWLER_SESSIONS: Dict[str, AsyncWebCrawler] = {}
-
-@tool("quick_crawl", "One-shot crawl for simple extraction. Returns markdown, HTML, or structured data.", {
-    "url": str,
-    "output_format": str,  # "markdown" | "html" | "structured" | "screenshot"
-    "extraction_schema": str,  # Optional: JSON schema for structured extraction
-    "js_code": str,  # Optional: JavaScript to execute before extraction
-    "wait_for": str,  # Optional: CSS selector to wait for
-})
-async def quick_crawl(args: Dict[str, Any]) -> Dict[str, Any]:
-    """Fast single-page crawl without session management."""
-    
-    crawler_config = BrowserConfig(headless=True, verbose=False)
-    run_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        js_code=args.get("js_code"),
-        wait_for=args.get("wait_for"),
-    )
-    
-    # Add extraction strategy if structured data requested
-    if args.get("extraction_schema"):
-        run_config.extraction_strategy = LLMExtractionStrategy(
-            provider="openai/gpt-4o-mini",
-            schema=json.loads(args["extraction_schema"]),
-            instruction="Extract data according to the provided schema."
-        )
-    
-    async with AsyncWebCrawler(config=crawler_config) as crawler:
-        result = await crawler.arun(url=args["url"], config=run_config)
-        
-        if not result.success:
-            return {
-                "content": [{
-                    "type": "text",
-                    "text": json.dumps({"error": result.error_message, "success": False})
-                }]
-            }
-        
-        output_map = {
-            "markdown": result.markdown_v2.raw_markdown if result.markdown_v2 else "",
-            "html": result.html,
-            "structured": result.extracted_content,
-            "screenshot": result.screenshot,
-        }
-        
-        response = {
-            "success": True,
-            "url": result.url,
-            "data": output_map.get(args["output_format"], result.markdown_v2.raw_markdown)
-        }
-        
-        return {"content": [{"type": "text", "text": json.dumps(response, indent=2)}]}
-
-
-@tool("start_session", "Start a persistent browser session for multi-step crawling and automation.", {
-    "session_id": str,
-    "headless": bool,  # Default True
-})
-async def start_session(args: Dict[str, Any]) -> Dict[str, Any]:
-    """Initialize a persistent crawler session."""
-    
-    session_id = args["session_id"]
-    if session_id in CRAWLER_SESSIONS:
-        return {"content": [{"type": "text", "text": json.dumps({
-            "error": f"Session {session_id} already exists",
-            "success": False
-        })}]}
-    
-    crawler_config = BrowserConfig(
-        headless=args.get("headless", True),
-        verbose=False
-    )
-    
-    crawler = AsyncWebCrawler(config=crawler_config)
-    await crawler.__aenter__()
-    CRAWLER_SESSIONS[session_id] = crawler
-    
-    return {"content": [{"type": "text", "text": json.dumps({
-        "success": True,
-        "session_id": session_id,
-        "message": f"Browser session {session_id} started"
-    })}]}
-
-
-@tool("navigate", "Navigate to a URL in an active session.", {
-    "session_id": str,
-    "url": str,
-    "wait_for": str,  # Optional: CSS selector to wait for
-    "js_code": str,  # Optional: JavaScript to execute after load
-})
-async def navigate(args: Dict[str, Any]) -> Dict[str, Any]:
-    """Navigate to URL in session."""
-    
-    session_id = args["session_id"]
-    if session_id not in CRAWLER_SESSIONS:
-        return {"content": [{"type": "text", "text": json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        })}]}
-    
-    crawler = CRAWLER_SESSIONS[session_id]
-    run_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        wait_for=args.get("wait_for"),
-        js_code=args.get("js_code"),
-    )
-    
-    result = await crawler.arun(url=args["url"], config=run_config)
-    
-    return {"content": [{"type": "text", "text": json.dumps({
-        "success": result.success,
-        "url": result.url,
-        "message": f"Navigated to {args['url']}"
-    })}]}
-
-
-@tool("extract_data", "Extract data from current page in session using schema or return markdown.", {
-    "session_id": str,
-    "output_format": str,  # "markdown" | "structured"
-    "extraction_schema": str,  # Required for structured, JSON schema
-    "wait_for": str,  # Optional: Wait for element before extraction
-    "js_code": str,  # Optional: Execute JS before extraction
-})
-async def extract_data(args: Dict[str, Any]) -> Dict[str, Any]:
-    """Extract data from current page."""
-    
-    session_id = args["session_id"]
-    if session_id not in CRAWLER_SESSIONS:
-        return {"content": [{"type": "text", "text": json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        })}]}
-    
-    crawler = CRAWLER_SESSIONS[session_id]
-    run_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        wait_for=args.get("wait_for"),
-        js_code=args.get("js_code"),
-    )
-    
-    if args["output_format"] == "structured" and args.get("extraction_schema"):
-        run_config.extraction_strategy = LLMExtractionStrategy(
-            provider="openai/gpt-4o-mini",
-            schema=json.loads(args["extraction_schema"]),
-            instruction="Extract data according to schema."
-        )
-    
-    result = await crawler.arun(config=run_config)
-    
-    if not result.success:
-        return {"content": [{"type": "text", "text": json.dumps({
-            "error": result.error_message,
-            "success": False
-        })}]}
-    
-    data = (result.extracted_content if args["output_format"] == "structured" 
-            else result.markdown_v2.raw_markdown if result.markdown_v2 else "")
-    
-    return {"content": [{"type": "text", "text": json.dumps({
-        "success": True,
-        "data": data
-    }, indent=2)}]}
-
-
-@tool("execute_js", "Execute JavaScript in the current page context.", {
-    "session_id": str,
-    "js_code": str,
-    "wait_for": str,  # Optional: Wait for element after execution
-})
-async def execute_js(args: Dict[str, Any]) -> Dict[str, Any]:
-    """Execute JavaScript in session."""
-    
-    session_id = args["session_id"]
-    if session_id not in CRAWLER_SESSIONS:
-        return {"content": [{"type": "text", "text": json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        })}]}
-    
-    crawler = CRAWLER_SESSIONS[session_id]
-    run_config = CrawlerRunConfig(
-        cache_mode=CacheMode.BYPASS,
-        js_code=args["js_code"],
-        wait_for=args.get("wait_for"),
-    )
-    
-    result = await crawler.arun(config=run_config)
-    
-    return {"content": [{"type": "text", "text": json.dumps({
-        "success": result.success,
-        "message": "JavaScript executed"
-    })}]}
-
-
-@tool("screenshot", "Take a screenshot of the current page.", {
-    "session_id": str,
-})
-async def screenshot(args: Dict[str, Any]) -> Dict[str, Any]:
-    """Capture screenshot."""
-    
-    session_id = args["session_id"]
-    if session_id not in CRAWLER_SESSIONS:
-        return {"content": [{"type": "text", "text": json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        })}]}
-    
-    crawler = CRAWLER_SESSIONS[session_id]
-    result = await crawler.arun(config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS))
-    
-    return {"content": [{"type": "text", "text": json.dumps({
-        "success": True,
-        "screenshot": result.screenshot if result.success else None
-    })}]}
-
-
-@tool("close_session", "Close and cleanup a browser session.", {
-    "session_id": str,
-})
-async def close_session(args: Dict[str, Any]) -> Dict[str, Any]:
-    """Close crawler session."""
-    
-    session_id = args["session_id"]
-    if session_id not in CRAWLER_SESSIONS:
-        return {"content": [{"type": "text", "text": json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        })}]}
-    
-    crawler = CRAWLER_SESSIONS.pop(session_id)
-    await crawler.__aexit__(None, None, None)
-    
-    return {"content": [{"type": "text", "text": json.dumps({
-        "success": True,
-        "message": f"Session {session_id} closed"
-    })}]}
-
-
-# Export all tools
-CRAWL_TOOLS = [
-    quick_crawl,
-    start_session,
-    navigate,
-    extract_data,
-    execute_js,
-    screenshot,
-    close_session,
-]
-```
-
-```python
-# c4ai_prompts.py
-"""System prompts for Crawl4AI agent."""
-
-SYSTEM_PROMPT = """You are an expert web crawling and browser automation agent powered by Crawl4AI.
-
-# Core Capabilities
-
-You can perform sophisticated multi-step web scraping and automation tasks through two modes:
-
-## Quick Mode (simple tasks)
-- Use `quick_crawl` for single-page data extraction
-- Best for: simple scrapes, getting page content, one-time extractions
-
-## Session Mode (complex tasks)
-- Use `start_session` to create persistent browser sessions
-- Navigate, interact, extract data across multiple pages
-- Essential for: workflows requiring JS execution, pagination, filtering, multi-step automation
-
-# Tool Usage Patterns
-
-## Simple Extraction
-1. Use `quick_crawl` with appropriate output_format
-2. Provide extraction_schema for structured data
-
-## Multi-Step Workflow
-1. `start_session` - Create browser session with unique ID
-2. `navigate` - Go to target URL
-3. `execute_js` - Interact with page (click buttons, scroll, fill forms)
-4. `extract_data` - Get data using schema or markdown
-5. Repeat steps 2-4 as needed
-6. `close_session` - Clean up when done
-
-# Critical Instructions
-
-1. **Iteration & Validation**: When tasks require filtering or conditional logic:
-   - Extract data first, analyze results
-   - Filter/validate in your reasoning
-   - Make subsequent tool calls based on validation
-   - Continue until task criteria are met
-
-2. **Structured Extraction**: Always use JSON schemas for structured data:
-   ```json
-   {
-     "type": "object",
-     "properties": {
-       "field_name": {"type": "string"},
-       "price": {"type": "number"}
-     }
-   }
-   ```
-
-3. **Session Management**:
-   - Generate unique session IDs (e.g., "product_scrape_001")
-   - Always close sessions when done
-   - Use sessions for tasks requiring multiple page visits
-
-4. **JavaScript Execution**:
-   - Use for: clicking buttons, scrolling, waiting for dynamic content
-   - Example: `js_code: "document.querySelector('.load-more').click()"`
-   - Combine with `wait_for` to ensure content loads
-
-5. **Error Handling**:
-   - Check `success` field in all responses
-   - Retry with different strategies if extraction fails
-   - Report specific errors to user
-
-6. **Data Persistence**:
-   - Save results using `Write` tool to JSON files
-   - Use descriptive filenames with timestamps
-   - Structure data clearly for user consumption
-
-# Example Workflows
-
-## Workflow 1: Filter & Crawl
-Task: "Find products >$10, crawl each, extract details"
-
-1. `quick_crawl` product listing page with schema for [name, price, url]
-2. Analyze results, filter price > 10 in reasoning
-3. `start_session` for detailed crawling
-4. For each filtered product:
-   - `navigate` to product URL
-   - `extract_data` with detail schema
-5. Aggregate results
-6. `close_session`
-7. `Write` results to JSON
-
-## Workflow 2: Paginated Scraping
-Task: "Scrape all items across multiple pages"
-
-1. `start_session`
-2. `navigate` to page 1
-3. `extract_data` items from current page
-4. Check for "next" button
-5. `execute_js` to click next
-6. Repeat 3-5 until no more pages
-7. `close_session`
-8. Save aggregated data
-
-## Workflow 3: Dynamic Content
-Task: "Scrape reviews after clicking 'Load More'"
-
-1. `start_session`
-2. `navigate` to product page
-3. `execute_js` to click load more button
-4. `wait_for` reviews container
-5. `extract_data` all reviews
-6. `close_session`
-
-# Quality Guidelines
-
-- **Be thorough**: Don't stop until task requirements are fully met
-- **Validate data**: Check extracted data matches expected format
-- **Handle edge cases**: Empty results, pagination limits, rate limiting
-- **Clear reporting**: Summarize what was found, any issues encountered
-- **Efficient**: Use quick_crawl when possible, sessions only when needed
-
-# Output Format
-
-When saving data, use clean JSON structure:
-```json
-{
-  "metadata": {
-    "scraped_at": "ISO timestamp",
-    "source_url": "...",
-    "total_items": 0
-  },
-  "data": [...]
-}
-```
-
-Always provide a final summary of:
-- Items found/processed
-- Time taken
-- Files created
-- Any warnings/errors
-
-Remember: You have unlimited turns to complete the task. Take your time, validate each step, and ensure quality results."""
-```
-
-```python
-# agent_crawl.py
-"""Crawl4AI Agent CLI - Browser automation agent powered by Claude Code SDK."""
-
-import asyncio
-import sys
-import json
-import uuid
-from pathlib import Path
-from datetime import datetime
-from typing import Optional
-import argparse
-
-from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions, create_sdk_mcp_server
-from claude_agent_sdk import AssistantMessage, TextBlock, ResultMessage
-
-from c4ai_tools import CRAWL_TOOLS
-from c4ai_prompts import SYSTEM_PROMPT
-
-
-class SessionStorage:
-    """Manage session storage in ~/.crawl4ai/agents/projects/"""
-    
-    def __init__(self, cwd: Optional[str] = None):
-        self.cwd = Path(cwd) if cwd else Path.cwd()
-        self.base_dir = Path.home() / ".crawl4ai" / "agents" / "projects"
-        self.project_dir = self.base_dir / self._sanitize_path(str(self.cwd.resolve()))
-        self.project_dir.mkdir(parents=True, exist_ok=True)
-        self.session_id = str(uuid.uuid4())
-        self.log_file = self.project_dir / f"{self.session_id}.jsonl"
-    
-    @staticmethod
-    def _sanitize_path(path: str) -> str:
-        """Convert /Users/unclecode/devs/test to -Users-unclecode-devs-test"""
-        return path.replace("/", "-").replace("\\", "-")
-    
-    def log(self, event_type: str, data: dict):
-        """Append event to JSONL log."""
-        entry = {
-            "timestamp": datetime.utcnow().isoformat(),
-            "event": event_type,
-            "session_id": self.session_id,
-            "data": data
-        }
-        with open(self.log_file, "a") as f:
-            f.write(json.dumps(entry) + "\n")
-    
-    def get_session_path(self) -> str:
-        """Return path to current session log."""
-        return str(self.log_file)
-
-
-class CrawlAgent:
-    """Crawl4AI agent wrapper."""
-    
-    def __init__(self, args: argparse.Namespace):
-        self.args = args
-        self.storage = SessionStorage(args.add_dir[0] if args.add_dir else None)
-        self.client: Optional[ClaudeSDKClient] = None
-        
-        # Create MCP server with crawl tools
-        self.crawler_server = create_sdk_mcp_server(
-            name="crawl4ai",
-            version="1.0.0",
-            tools=CRAWL_TOOLS
-        )
-        
-        # Build options
-        self.options = ClaudeAgentOptions(
-            mcp_servers={"crawler": self.crawler_server},
-            allowed_tools=[
-                "mcp__crawler__quick_crawl",
-                "mcp__crawler__start_session",
-                "mcp__crawler__navigate",
-                "mcp__crawler__extract_data",
-                "mcp__crawler__execute_js",
-                "mcp__crawler__screenshot",
-                "mcp__crawler__close_session",
-                "Write", "Read", "Bash"
-            ],
-            system_prompt=SYSTEM_PROMPT if not args.system_prompt else args.system_prompt,
-            permission_mode=args.permission_mode or "acceptEdits",
-            cwd=args.add_dir[0] if args.add_dir else str(Path.cwd()),
-            model=args.model,
-            session_id=args.session_id or self.storage.session_id,
-        )
-    
-    async def run(self, prompt: str):
-        """Execute crawl task."""
-        
-        self.storage.log("session_start", {
-            "prompt": prompt,
-            "cwd": self.options.cwd,
-            "model": self.options.model
-        })
-        
-        print(f"\n🕷️  Crawl4AI Agent")
-        print(f"📁 Session: {self.storage.session_id}")
-        print(f"💾 Log: {self.storage.get_session_path()}")
-        print(f"🎯 Task: {prompt}\n")
-        
-        async with ClaudeSDKClient(options=self.options) as client:
-            self.client = client
-            await client.query(prompt)
-            
-            turn = 0
-            async for message in client.receive_messages():
-                turn += 1
-                
-                if isinstance(message, AssistantMessage):
-                    for block in message.content:
-                        if isinstance(block, TextBlock):
-                            print(f"\n💭 [{turn}] {block.text}")
-                            self.storage.log("assistant_message", {"turn": turn, "text": block.text})
-                
-                elif isinstance(message, ResultMessage):
-                    print(f"\n✅ Completed in {message.duration_ms/1000:.2f}s")
-                    print(f"💰 Cost: ${message.total_cost_usd:.4f}" if message.total_cost_usd else "")
-                    print(f"🔄 Turns: {message.num_turns}")
-                    
-                    self.storage.log("session_end", {
-                        "duration_ms": message.duration_ms,
-                        "cost_usd": message.total_cost_usd,
-                        "turns": message.num_turns,
-                        "success": not message.is_error
-                    })
-                    break
-        
-        print(f"\n📊 Session log: {self.storage.get_session_path()}\n")
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Crawl4AI Agent - Browser automation powered by Claude Code SDK",
-        formatter_class=argparse.RawDescriptionHelpFormatter
-    )
-    
-    parser.add_argument("prompt", nargs="?", help="Your crawling task prompt")
-    parser.add_argument("--system-prompt", help="Custom system prompt")
-    parser.add_argument("--permission-mode", choices=["acceptEdits", "bypassPermissions", "default", "plan"],
-                       help="Permission mode for tool execution")
-    parser.add_argument("--model", help="Model to use (e.g., 'sonnet', 'opus')")
-    parser.add_argument("--add-dir", nargs="+", help="Additional directories for file access")
-    parser.add_argument("--session-id", help="Use specific session ID (UUID)")
-    parser.add_argument("-v", "--version", action="version", version="Crawl4AI Agent 1.0.0")
-    parser.add_argument("--debug", action="store_true", help="Enable debug mode")
-    
-    args = parser.parse_args()
-    
-    if not args.prompt:
-        parser.print_help()
-        print("\nExample usage:")
-        print('  crawl-agent "Scrape all products from example.com with price > $10"')
-        print('  crawl-agent --add-dir ~/projects "Find all Python files and analyze imports"')
-        sys.exit(1)
-    
-    try:
-        agent = CrawlAgent(args)
-        asyncio.run(agent.run(args.prompt))
-    except KeyboardInterrupt:
-        print("\n\n⚠️  Interrupted by user")
-        sys.exit(0)
-    except Exception as e:
-        print(f"\n❌ Error: {e}")
-        if args.debug:
-            raise
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()
-```
-
-**Usage:**
-
-```bash
-# Simple scrape
-python agent_crawl.py "Get all product names from example.com"
-
-# Complex filtering
-python agent_crawl.py "Find products >$10 from shop.com, crawl each, extract id/name/price"
-
-# Multi-step automation
-python agent_crawl.py "Go to amazon.com, search 'laptop', filter 4+ stars, scrape top 10"
-
-# With options
-python agent_crawl.py --add-dir ~/projects --model sonnet "Scrape competitor prices"
-```
-
-**Session logs stored at:**
-`~/.crawl4ai/agents/projects/-Users-unclecode-devs-test/{uuid}.jsonl`

crawl4ai/agent/agent_crawl.py

@@ -1,126 +0,0 @@
-# agent_crawl.py
-"""Crawl4AI Agent CLI - Browser automation agent powered by OpenAI Agents SDK."""
-
-import asyncio
-import sys
-import os
-import argparse
-from pathlib import Path
-
-from agents import Agent, Runner, set_default_openai_key
-
-from .crawl_tools import CRAWL_TOOLS
-from .crawl_prompts import SYSTEM_PROMPT
-from .browser_manager import BrowserManager
-from .terminal_ui import TerminalUI
-
-
-class CrawlAgent:
-    """Crawl4AI agent wrapper using OpenAI Agents SDK."""
-
-    def __init__(self, args: argparse.Namespace):
-        self.args = args
-        self.ui = TerminalUI()
-
-        # Set API key
-        api_key = os.getenv("OPENAI_API_KEY")
-        if not api_key:
-            raise ValueError("OPENAI_API_KEY environment variable not set")
-        set_default_openai_key(api_key)
-
-        # Create agent
-        self.agent = Agent(
-            name="Crawl4AI Agent",
-            instructions=SYSTEM_PROMPT,
-            model=args.model or "gpt-4.1",
-            tools=CRAWL_TOOLS,
-            tool_use_behavior="run_llm_again",  # CRITICAL: Run LLM again after tools to generate response
-        )
-
-    async def run_single_shot(self, prompt: str):
-        """Execute a single crawl task."""
-        self.ui.console.print(f"\n🕷️  [bold cyan]Crawl4AI Agent[/bold cyan]")
-        self.ui.console.print(f"🎯 Task: {prompt}\n")
-
-        try:
-            result = await Runner.run(
-                starting_agent=self.agent,
-                input=prompt,
-                context=None,
-                max_turns=100,  # Allow up to 100 turns for complex tasks
-            )
-
-            self.ui.console.print(f"\n[bold green]Result:[/bold green]")
-            self.ui.console.print(result.final_output)
-
-            if hasattr(result, 'usage'):
-                self.ui.console.print(f"\n[dim]Tokens: {result.usage}[/dim]")
-
-        except Exception as e:
-            self.ui.print_error(f"Error: {e}")
-            if self.args.debug:
-                raise
-
-    async def run_chat_mode(self):
-        """Run interactive chat mode with streaming visibility."""
-        from .chat_mode import ChatMode
-
-        chat = ChatMode(self.agent, self.ui)
-        await chat.run()
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Crawl4AI Agent - Browser automation powered by OpenAI Agents SDK",
-        formatter_class=argparse.RawDescriptionHelpFormatter
-    )
-
-    parser.add_argument("prompt", nargs="?", help="Your crawling task prompt (not used in --chat mode)")
-    parser.add_argument("--chat", action="store_true", help="Start interactive chat mode")
-    parser.add_argument("--model", help="Model to use (e.g., 'gpt-4.1', 'gpt-5-nano')", default="gpt-4.1")
-    parser.add_argument("-v", "--version", action="version", version="Crawl4AI Agent 2.0.0")
-    parser.add_argument("--debug", action="store_true", help="Enable debug mode")
-
-    args = parser.parse_args()
-
-    # Chat mode - interactive
-    if args.chat:
-        try:
-            agent = CrawlAgent(args)
-            asyncio.run(agent.run_chat_mode())
-        except KeyboardInterrupt:
-            print("\n\n⚠️  Chat interrupted by user")
-            sys.exit(0)
-        except Exception as e:
-            print(f"\n❌ Error: {e}")
-            if args.debug:
-                raise
-            sys.exit(1)
-        return
-
-    # Single-shot mode - requires prompt
-    if not args.prompt:
-        parser.print_help()
-        print("\nExample usage:")
-        print('  # Single-shot mode:')
-        print('  python -m crawl4ai.agent.agent_crawl "Scrape products from example.com"')
-        print()
-        print('  # Interactive chat mode:')
-        print('  python -m crawl4ai.agent.agent_crawl --chat')
-        sys.exit(1)
-
-    try:
-        agent = CrawlAgent(args)
-        asyncio.run(agent.run_single_shot(args.prompt))
-    except KeyboardInterrupt:
-        print("\n\n⚠️  Interrupted by user")
-        sys.exit(0)
-    except Exception as e:
-        print(f"\n❌ Error: {e}")
-        if args.debug:
-            raise
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

crawl4ai/agent/browser_manager.py

@@ -1,73 +0,0 @@
-"""Browser session management with singleton pattern for persistent browser instances."""
-
-from typing import Optional
-from crawl4ai import AsyncWebCrawler, BrowserConfig
-
-
-class BrowserManager:
-    """Singleton browser manager for persistent browser sessions across agent operations."""
-
-    _instance: Optional['BrowserManager'] = None
-    _crawler: Optional[AsyncWebCrawler] = None
-    _config: Optional[BrowserConfig] = None
-
-    def __new__(cls):
-        if cls._instance is None:
-            cls._instance = super().__new__(cls)
-        return cls._instance
-
-    @classmethod
-    async def get_browser(cls, config: Optional[BrowserConfig] = None) -> AsyncWebCrawler:
-        """
-        Get or create the singleton browser instance.
-
-        Args:
-            config: Optional browser configuration. Only used if no browser exists yet.
-                   To change config, use reconfigure_browser() instead.
-
-        Returns:
-            AsyncWebCrawler instance
-        """
-        # Create new browser if needed
-        if cls._crawler is None:
-            # Create default config if none provided
-            if config is None:
-                config = BrowserConfig(headless=True, verbose=False)
-
-            cls._crawler = AsyncWebCrawler(config=config)
-            await cls._crawler.start()
-            cls._config = config
-
-        return cls._crawler
-
-    @classmethod
-    async def reconfigure_browser(cls, new_config: BrowserConfig) -> AsyncWebCrawler:
-        """
-        Close current browser and create a new one with different configuration.
-
-        Args:
-            new_config: New browser configuration
-
-        Returns:
-            New AsyncWebCrawler instance
-        """
-        await cls.close_browser()
-        return await cls.get_browser(new_config)
-
-    @classmethod
-    async def close_browser(cls):
-        """Close the current browser instance and cleanup."""
-        if cls._crawler is not None:
-            await cls._crawler.close()
-            cls._crawler = None
-            cls._config = None
-
-    @classmethod
-    def is_browser_active(cls) -> bool:
-        """Check if browser is currently active."""
-        return cls._crawler is not None
-
-    @classmethod
-    def get_current_config(cls) -> Optional[BrowserConfig]:
-        """Get the current browser configuration."""
-        return cls._config

crawl4ai/agent/chat_mode.py

@@ -1,213 +0,0 @@
-# chat_mode.py
-"""Interactive chat mode with streaming visibility for Crawl4AI Agent."""
-
-import asyncio
-from typing import Optional
-from agents import Agent, Runner
-
-from .terminal_ui import TerminalUI
-from .browser_manager import BrowserManager
-
-
-class ChatMode:
-    """Interactive chat mode with real-time status updates and tool visibility."""
-
-    def __init__(self, agent: Agent, ui: TerminalUI):
-        self.agent = agent
-        self.ui = ui
-        self._exit_requested = False
-        self.conversation_history = []  # Track full conversation for context
-
-        # Generate unique session ID
-        import time
-        self.session_id = f"session_{int(time.time())}"
-
-    async def _handle_command(self, command: str) -> bool:
-        """Handle special chat commands.
-
-        Returns:
-            True if command was /exit, False otherwise
-        """
-        cmd = command.lower().strip()
-
-        if cmd == '/exit' or cmd == '/quit':
-            self._exit_requested = True
-            self.ui.print_info("Exiting chat mode...")
-            return True
-
-        elif cmd == '/clear':
-            self.ui.clear_screen()
-            self.ui.show_header(session_id=self.session_id)
-            return False
-
-        elif cmd == '/help':
-            self.ui.show_commands()
-            return False
-
-        elif cmd == '/browser':
-            # Show browser status
-            if BrowserManager.is_browser_active():
-                config = BrowserManager.get_current_config()
-                self.ui.print_info(f"Browser active: headless={config.headless if config else 'unknown'}")
-            else:
-                self.ui.print_info("No browser instance active")
-            return False
-
-        else:
-            self.ui.print_error(f"Unknown command: {command}")
-            self.ui.print_info("Available commands: /exit, /clear, /help, /browser")
-            return False
-
-    async def run(self):
-        """Run the interactive chat loop with streaming responses and visibility."""
-        # Show header with session ID (tips are now inside)
-        self.ui.show_header(session_id=self.session_id)
-
-        try:
-            while not self._exit_requested:
-                # Get user input
-                try:
-                    user_input = await asyncio.to_thread(self.ui.get_user_input)
-                except EOFError:
-                    break
-
-                # Handle commands
-                if user_input.startswith('/'):
-                    should_exit = await self._handle_command(user_input)
-                    if should_exit:
-                        break
-                    continue
-
-                # Skip empty input
-                if not user_input.strip():
-                    continue
-
-                # Add user message to conversation history
-                self.conversation_history.append({
-                    "role": "user",
-                    "content": user_input
-                })
-
-                # Show thinking indicator
-                self.ui.console.print("\n[cyan]Agent:[/cyan] [dim italic]thinking...[/dim italic]")
-
-                try:
-                    # Run agent with streaming, passing conversation history for context
-                    result = Runner.run_streamed(
-                        self.agent,
-                        input=self.conversation_history,  # Pass full conversation history
-                        context=None,
-                        max_turns=100,  # Allow up to 100 turns for complex multi-step tasks
-                    )
-
-                    # Track what we've seen
-                    response_text = []
-                    tools_called = []
-                    current_tool = None
-
-                    # Process streaming events
-                    async for event in result.stream_events():
-                        # DEBUG: Print all event types
-                        # self.ui.console.print(f"[dim]DEBUG: event type={event.type}[/dim]")
-
-                        # Agent switched
-                        if event.type == "agent_updated_stream_event":
-                            self.ui.console.print(f"\n[dim]→ Agent: {event.new_agent.name}[/dim]")
-
-                        # Items generated (tool calls, outputs, text)
-                        elif event.type == "run_item_stream_event":
-                            item = event.item
-
-                            # Tool call started
-                            if item.type == "tool_call_item":
-                                # Get tool name from raw_item
-                                current_tool = item.raw_item.name if hasattr(item.raw_item, 'name') else "unknown"
-                                tools_called.append(current_tool)
-
-                                # Show tool name and args clearly
-                                tool_display = current_tool
-                                self.ui.console.print(f"\n[yellow]🔧 Calling:[/yellow] [bold]{tool_display}[/bold]")
-
-                                # Show tool arguments if present
-                                if hasattr(item.raw_item, 'arguments'):
-                                    try:
-                                        import json
-                                        args_str = item.raw_item.arguments
-                                        args = json.loads(args_str) if isinstance(args_str, str) else args_str
-                                        # Show key args only
-                                        key_args = {k: v for k, v in args.items() if k in ['url', 'session_id', 'output_format']}
-                                        if key_args:
-                                            params_str = ", ".join(f"{k}={v}" for k, v in key_args.items())
-                                            self.ui.console.print(f"  [dim]({params_str})[/dim]")
-                                    except:
-                                        pass
-
-                            # Tool output received
-                            elif item.type == "tool_call_output_item":
-                                if current_tool:
-                                    self.ui.console.print(f"  [green]✓[/green] [dim]completed[/dim]")
-                                    current_tool = None
-
-                            # Agent text response (multiple types)
-                            elif item.type == "text_item":
-                                # Clear "thinking..." line if this is first text
-                                if not response_text:
-                                    self.ui.console.print("\r[cyan]Agent:[/cyan] ", end="")
-
-                                # Stream the text
-                                self.ui.console.print(item.text, end="")
-                                response_text.append(item.text)
-
-                            # Message output (final response)
-                            elif item.type == "message_output_item":
-                                # This is the final formatted response
-                                if not response_text:
-                                    self.ui.console.print("\n[cyan]Agent:[/cyan] ", end="")
-
-                                # Extract text from content blocks
-                                if hasattr(item.raw_item, 'content') and item.raw_item.content:
-                                    for content_block in item.raw_item.content:
-                                        if hasattr(content_block, 'text'):
-                                            text = content_block.text
-                                            self.ui.console.print(text, end="")
-                                            response_text.append(text)
-
-                        # Text deltas (real-time streaming)
-                        elif event.type == "text_delta_stream_event":
-                            # Clear "thinking..." if this is first delta
-                            if not response_text:
-                                self.ui.console.print("\r[cyan]Agent:[/cyan] ", end="")
-
-                            # Stream character by character for responsiveness
-                            self.ui.console.print(event.delta, end="", markup=False)
-                            response_text.append(event.delta)
-
-                    # Newline after response
-                    self.ui.console.print()
-
-                    # Show summary after response
-                    if tools_called:
-                        self.ui.console.print(f"\n[dim]Tools used: {', '.join(set(tools_called))}[/dim]")
-
-                    # Add agent response to conversation history
-                    if response_text:
-                        agent_response = "".join(response_text)
-                        self.conversation_history.append({
-                            "role": "assistant",
-                            "content": agent_response
-                        })
-
-                except Exception as e:
-                    self.ui.print_error(f"Error during agent execution: {e}")
-                    import traceback
-                    traceback.print_exc()
-
-        except KeyboardInterrupt:
-            self.ui.print_info("\n\nChat interrupted by user")
-
-        finally:
-            # Cleanup browser on exit
-            self.ui.console.print("\n[dim]Cleaning up...[/dim]")
-            await BrowserManager.close_browser()
-            self.ui.print_info("Browser closed")
-            self.ui.console.print("[bold green]Goodbye![/bold green]\n")

crawl4ai/agent/crawl_prompts.py

@@ -1,142 +0,0 @@
-# crawl_prompts.py
-"""System prompts for Crawl4AI agent."""
-
-SYSTEM_PROMPT = """You are an expert web crawling and browser automation agent powered by Crawl4AI.
-
-# Core Capabilities
-
-You can perform sophisticated multi-step web scraping and automation tasks through two modes:
-
-## Quick Mode (simple tasks)
-- Use `quick_crawl` for single-page data extraction
-- Best for: simple scrapes, getting page content, one-time extractions
-- Returns markdown or HTML content immediately
-
-## Session Mode (complex tasks)
-- Use `start_session` to create persistent browser sessions
-- Navigate, interact, extract data across multiple pages
-- Essential for: workflows requiring JS execution, pagination, filtering, multi-step automation
-- ALWAYS close sessions with `close_session` when done
-
-# Tool Usage Patterns
-
-## Simple Extraction
-1. Use `quick_crawl` with appropriate output_format (markdown or html)
-2. Provide extraction_schema for structured data if needed
-
-## Multi-Step Workflow
-1. `start_session` - Create browser session with unique ID
-2. `navigate` - Go to target URL
-3. `execute_js` - Interact with page (click buttons, scroll, fill forms)
-4. `extract_data` - Get data using schema or markdown
-5. Repeat steps 2-4 as needed
-6. `close_session` - REQUIRED - Clean up when done
-
-# Critical Instructions
-
-1. **Session Management - CRITICAL**:
-   - Generate unique session IDs (e.g., "product_scrape_001")
-   - ALWAYS close sessions when done using `close_session`
-   - Use sessions for tasks requiring multiple page visits
-   - Track which session you're using
-
-2. **JavaScript Execution**:
-   - Use for: clicking buttons, scrolling, waiting for dynamic content
-   - Example: `js_code: "document.querySelector('.load-more').click()"`
-   - Combine with `wait_for` to ensure content loads
-
-3. **Error Handling**:
-   - Check `success` field in all tool responses
-   - If a tool fails, analyze why and try alternative approach
-   - Report specific errors to user
-   - Don't give up - try different strategies
-
-4. **Structured Extraction**: Use JSON schemas for structured data:
-   ```json
-   {
-     "type": "object",
-     "properties": {
-       "field_name": {"type": "string"},
-       "price": {"type": "number"}
-     }
-   }
-   ```
-
-# Example Workflows
-
-## Workflow 1: Simple Multi-Page Crawl
-Task: "Crawl example.com and example.org, extract titles"
-
-```
-Step 1: Crawl both pages
-- Use quick_crawl(url="https://example.com", output_format="markdown")
-- Use quick_crawl(url="https://example.org", output_format="markdown")
-- Extract titles from markdown content
-
-Step 2: Report
-- Summarize the titles found
-```
-
-## Workflow 2: Session-Based Extraction
-Task: "Start session, navigate, extract, save"
-
-```
-Step 1: Create and navigate
-- start_session(session_id="extract_001")
-- navigate(session_id="extract_001", url="https://example.com")
-
-Step 2: Extract content
-- extract_data(session_id="extract_001", output_format="markdown")
-- Report the extracted content to user
-
-Step 3: Cleanup (REQUIRED)
-- close_session(session_id="extract_001")
-```
-
-## Workflow 3: Error Recovery
-Task: "Handle failed crawl gracefully"
-
-```
-Step 1: Attempt crawl
-- quick_crawl(url="https://invalid-site.com")
-- Check success field in response
-
-Step 2: On failure
-- Acknowledge the error to user
-- Provide clear error message
-- DON'T give up - suggest alternative or retry
-
-Step 3: Continue with valid request
-- quick_crawl(url="https://example.com")
-- Complete the task successfully
-```
-
-## Workflow 4: Paginated Scraping
-Task: "Scrape all items across multiple pages"
-
-1. `start_session`
-2. `navigate` to page 1
-3. `extract_data` items from current page
-4. Check for "next" button
-5. `execute_js` to click next
-6. Repeat 3-5 until no more pages
-7. `close_session` (REQUIRED)
-8. Report aggregated data
-
-# Quality Guidelines
-
-- **Be thorough**: Don't stop until task requirements are fully met
-- **Validate data**: Check extracted data matches expected format
-- **Handle edge cases**: Empty results, pagination limits, rate limiting
-- **Clear reporting**: Summarize what was found, any issues encountered
-- **Efficient**: Use quick_crawl when possible, sessions only when needed
-- **Session cleanup**: ALWAYS close sessions you created
-
-# Key Reminders
-
-1. **Sessions**: Always close what you open
-2. **Errors**: Handle gracefully, don't stop at first failure
-3. **Validation**: Check tool responses, verify success
-4. **Completion**: Confirm all steps done, report results clearly
-
-Remember: You have unlimited turns to complete the task. Take your time, validate each step, and ensure quality results."""

crawl4ai/agent/crawl_tools.py

@@ -1,362 +0,0 @@
-# crawl_tools.py
-"""Crawl4AI tools for OpenAI Agents SDK."""
-
-import json
-from typing import Any, Dict, Optional
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-from crawl4ai.extraction_strategy import LLMExtractionStrategy
-from agents import function_tool
-
-from .browser_manager import BrowserManager
-
-# Global session storage (for named sessions only)
-CRAWLER_SESSIONS: Dict[str, AsyncWebCrawler] = {}
-CRAWLER_SESSION_URLS: Dict[str, str] = {}  # Track current URL per session
-
-
-@function_tool
-async def quick_crawl(
-    url: str,
-    output_format: str = "markdown",
-    extraction_schema: Optional[str] = None,
-    js_code: Optional[str] = None,
-    wait_for: Optional[str] = None
-) -> str:
-    """One-shot crawl for simple extraction. Returns markdown, HTML, or structured data.
-
-    Args:
-        url: The URL to crawl
-        output_format: Output format - "markdown", "html", "structured", or "screenshot"
-        extraction_schema: Optional JSON schema for structured extraction
-        js_code: Optional JavaScript to execute before extraction
-        wait_for: Optional CSS selector to wait for
-
-    Returns:
-        JSON string with success status, url, and extracted data
-    """
-    # Use singleton browser manager
-    crawler_config = BrowserConfig(headless=True, verbose=False)
-    crawler = await BrowserManager.get_browser(crawler_config)
-
-    run_config = CrawlerRunConfig(
-        verbose=False,
-        cache_mode=CacheMode.BYPASS,
-        js_code=js_code,
-        wait_for=wait_for,
-    )
-
-    # Add extraction strategy if structured data requested
-    if extraction_schema:
-        run_config.extraction_strategy = LLMExtractionStrategy(
-            provider="openai/gpt-4o-mini",
-            schema=json.loads(extraction_schema),
-            instruction="Extract data according to the provided schema."
-        )
-
-    result = await crawler.arun(url=url, config=run_config)
-
-    if not result.success:
-        return json.dumps({
-            "error": result.error_message,
-            "success": False
-        }, indent=2)
-
-    # Handle markdown - can be string or MarkdownGenerationResult object
-    markdown_content = ""
-    if isinstance(result.markdown, str):
-        markdown_content = result.markdown
-    elif hasattr(result.markdown, 'raw_markdown'):
-        markdown_content = result.markdown.raw_markdown
-
-    output_map = {
-        "markdown": markdown_content,
-        "html": result.html,
-        "structured": result.extracted_content,
-        "screenshot": result.screenshot,
-    }
-
-    response = {
-        "success": True,
-        "url": result.url,
-        "data": output_map.get(output_format, markdown_content)
-    }
-
-    return json.dumps(response, indent=2)
-
-
-@function_tool
-async def start_session(
-    session_id: str,
-    headless: bool = True
-) -> str:
-    """Start a named browser session for multi-step crawling and automation.
-
-    Args:
-        session_id: Unique identifier for the session
-        headless: Whether to run browser in headless mode (default True)
-
-    Returns:
-        JSON string with success status and session info
-    """
-    if session_id in CRAWLER_SESSIONS:
-        return json.dumps({
-            "error": f"Session {session_id} already exists",
-            "success": False
-        }, indent=2)
-
-    # Use the singleton browser
-    crawler_config = BrowserConfig(
-        headless=headless,
-        verbose=False
-    )
-    crawler = await BrowserManager.get_browser(crawler_config)
-
-    # Store reference for named session
-    CRAWLER_SESSIONS[session_id] = crawler
-
-    return json.dumps({
-        "success": True,
-        "session_id": session_id,
-        "message": f"Browser session {session_id} started"
-    }, indent=2)
-
-
-@function_tool
-async def navigate(
-    session_id: str,
-    url: str,
-    wait_for: Optional[str] = None,
-    js_code: Optional[str] = None
-) -> str:
-    """Navigate to a URL in an active session.
-
-    Args:
-        session_id: The session identifier
-        url: The URL to navigate to
-        wait_for: Optional CSS selector to wait for
-        js_code: Optional JavaScript to execute after load
-
-    Returns:
-        JSON string with navigation result
-    """
-    if session_id not in CRAWLER_SESSIONS:
-        return json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        }, indent=2)
-
-    crawler = CRAWLER_SESSIONS[session_id]
-    run_config = CrawlerRunConfig(
-        verbose=False,
-        cache_mode=CacheMode.BYPASS,
-        wait_for=wait_for,
-        js_code=js_code,
-    )
-
-    result = await crawler.arun(url=url, config=run_config)
-
-    # Store current URL for this session
-    if result.success:
-        CRAWLER_SESSION_URLS[session_id] = result.url
-
-    return json.dumps({
-        "success": result.success,
-        "url": result.url,
-        "message": f"Navigated to {url}"
-    }, indent=2)
-
-
-@function_tool
-async def extract_data(
-    session_id: str,
-    output_format: str = "markdown",
-    extraction_schema: Optional[str] = None,
-    wait_for: Optional[str] = None,
-    js_code: Optional[str] = None
-) -> str:
-    """Extract data from current page in session using schema or return markdown.
-
-    Args:
-        session_id: The session identifier
-        output_format: "markdown" or "structured"
-        extraction_schema: Required for structured - JSON schema
-        wait_for: Optional - Wait for element before extraction
-        js_code: Optional - Execute JS before extraction
-
-    Returns:
-        JSON string with extracted data
-    """
-    if session_id not in CRAWLER_SESSIONS:
-        return json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        }, indent=2)
-
-    # Check if we have a current URL for this session
-    if session_id not in CRAWLER_SESSION_URLS:
-        return json.dumps({
-            "error": "No page loaded in session. Use 'navigate' first.",
-            "success": False
-        }, indent=2)
-
-    crawler = CRAWLER_SESSIONS[session_id]
-    current_url = CRAWLER_SESSION_URLS[session_id]
-
-    run_config = CrawlerRunConfig(
-        verbose=False,
-        cache_mode=CacheMode.BYPASS,
-        wait_for=wait_for,
-        js_code=js_code,
-    )
-
-    if output_format == "structured" and extraction_schema:
-        run_config.extraction_strategy = LLMExtractionStrategy(
-            provider="openai/gpt-4o-mini",
-            schema=json.loads(extraction_schema),
-            instruction="Extract data according to schema."
-        )
-
-    result = await crawler.arun(url=current_url, config=run_config)
-
-    if not result.success:
-        return json.dumps({
-            "error": result.error_message,
-            "success": False
-        }, indent=2)
-
-    # Handle markdown - can be string or MarkdownGenerationResult object
-    markdown_content = ""
-    if isinstance(result.markdown, str):
-        markdown_content = result.markdown
-    elif hasattr(result.markdown, 'raw_markdown'):
-        markdown_content = result.markdown.raw_markdown
-
-    data = (result.extracted_content if output_format == "structured"
-            else markdown_content)
-
-    return json.dumps({
-        "success": True,
-        "data": data
-    }, indent=2)
-
-
-@function_tool
-async def execute_js(
-    session_id: str,
-    js_code: str,
-    wait_for: Optional[str] = None
-) -> str:
-    """Execute JavaScript in the current page context.
-
-    Args:
-        session_id: The session identifier
-        js_code: JavaScript code to execute
-        wait_for: Optional - Wait for element after execution
-
-    Returns:
-        JSON string with execution result
-    """
-    if session_id not in CRAWLER_SESSIONS:
-        return json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        }, indent=2)
-
-    # Check if we have a current URL for this session
-    if session_id not in CRAWLER_SESSION_URLS:
-        return json.dumps({
-            "error": "No page loaded in session. Use 'navigate' first.",
-            "success": False
-        }, indent=2)
-
-    crawler = CRAWLER_SESSIONS[session_id]
-    current_url = CRAWLER_SESSION_URLS[session_id]
-
-    run_config = CrawlerRunConfig(
-        verbose=False,
-        cache_mode=CacheMode.BYPASS,
-        js_code=js_code,
-        wait_for=wait_for,
-    )
-
-    result = await crawler.arun(url=current_url, config=run_config)
-
-    return json.dumps({
-        "success": result.success,
-        "message": "JavaScript executed"
-    }, indent=2)
-
-
-@function_tool
-async def screenshot(session_id: str) -> str:
-    """Take a screenshot of the current page.
-
-    Args:
-        session_id: The session identifier
-
-    Returns:
-        JSON string with screenshot data
-    """
-    if session_id not in CRAWLER_SESSIONS:
-        return json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        }, indent=2)
-
-    # Check if we have a current URL for this session
-    if session_id not in CRAWLER_SESSION_URLS:
-        return json.dumps({
-            "error": "No page loaded in session. Use 'navigate' first.",
-            "success": False
-        }, indent=2)
-
-    crawler = CRAWLER_SESSIONS[session_id]
-    current_url = CRAWLER_SESSION_URLS[session_id]
-
-    result = await crawler.arun(
-        url=current_url,
-        config=CrawlerRunConfig(verbose=False, cache_mode=CacheMode.BYPASS, screenshot=True)
-    )
-
-    return json.dumps({
-        "success": True,
-        "screenshot": result.screenshot if result.success else None
-    }, indent=2)
-
-
-@function_tool
-async def close_session(session_id: str) -> str:
-    """Close and cleanup a named browser session.
-
-    Args:
-        session_id: The session identifier
-
-    Returns:
-        JSON string with closure confirmation
-    """
-    if session_id not in CRAWLER_SESSIONS:
-        return json.dumps({
-            "error": f"Session {session_id} not found",
-            "success": False
-        }, indent=2)
-
-    # Remove from named sessions, but don't close the singleton browser
-    CRAWLER_SESSIONS.pop(session_id)
-    CRAWLER_SESSION_URLS.pop(session_id, None)  # Remove URL tracking
-
-    return json.dumps({
-        "success": True,
-        "message": f"Session {session_id} closed"
-    }, indent=2)
-
-
-# Export all tools
-CRAWL_TOOLS = [
-    quick_crawl,
-    start_session,
-    navigate,
-    extract_data,
-    execute_js,
-    screenshot,
-    close_session,
-]

crawl4ai/agent/run_all_tests.py

@@ -1,321 +0,0 @@
-#!/usr/bin/env python
-"""
-Automated Test Suite Runner for Crawl4AI Agent
-Runs all tests in sequence: Component → Tools → Scenarios
-Generates comprehensive test report with timing and pass/fail metrics.
-"""
-
-import sys
-import asyncio
-import time
-import json
-from pathlib import Path
-from datetime import datetime
-from typing import Dict, Any, List
-
-# Add parent to path for imports
-sys.path.insert(0, str(Path(__file__).parent.parent.parent))
-
-
-class TestSuiteRunner:
-    """Orchestrates all test suites with reporting."""
-
-    def __init__(self, output_dir: Path):
-        self.output_dir = output_dir
-        self.output_dir.mkdir(exist_ok=True, parents=True)
-        self.results = {
-            "timestamp": datetime.now().isoformat(),
-            "test_suites": [],
-            "overall_status": "PENDING"
-        }
-
-    def print_banner(self, text: str, char: str = "="):
-        """Print a formatted banner."""
-        width = 70
-        print(f"\n{char * width}")
-        print(f"{text:^{width}}")
-        print(f"{char * width}\n")
-
-    async def run_component_tests(self) -> Dict[str, Any]:
-        """Run component tests (test_chat.py)."""
-        self.print_banner("TEST SUITE 1/3: COMPONENT TESTS", "=")
-        print("Testing: BrowserManager, TerminalUI, MCP Server, ChatMode")
-        print("Expected duration: ~5 seconds\n")
-
-        start_time = time.time()
-        suite_result = {
-            "name": "Component Tests",
-            "file": "test_chat.py",
-            "status": "PENDING",
-            "duration_seconds": 0,
-            "tests_run": 4,
-            "tests_passed": 0,
-            "tests_failed": 0,
-            "details": []
-        }
-
-        try:
-            # Import and run the test
-            from crawl4ai.agent import test_chat
-
-            # Capture the result
-            success = await test_chat.test_components()
-
-            duration = time.time() - start_time
-            suite_result["duration_seconds"] = duration
-
-            if success:
-                suite_result["status"] = "PASS"
-                suite_result["tests_passed"] = 4
-                print(f"\n✓ Component tests PASSED in {duration:.2f}s")
-            else:
-                suite_result["status"] = "FAIL"
-                suite_result["tests_failed"] = 4
-                print(f"\n✗ Component tests FAILED in {duration:.2f}s")
-
-        except Exception as e:
-            duration = time.time() - start_time
-            suite_result["status"] = "ERROR"
-            suite_result["error"] = str(e)
-            suite_result["duration_seconds"] = duration
-            suite_result["tests_failed"] = 4
-            print(f"\n✗ Component tests ERROR: {e}")
-
-        return suite_result
-
-    async def run_tool_tests(self) -> Dict[str, Any]:
-        """Run tool integration tests (test_tools.py)."""
-        self.print_banner("TEST SUITE 2/3: TOOL INTEGRATION TESTS", "=")
-        print("Testing: Quick crawl, Session workflow, HTML format")
-        print("Expected duration: ~30 seconds (uses browser)\n")
-
-        start_time = time.time()
-        suite_result = {
-            "name": "Tool Integration Tests",
-            "file": "test_tools.py",
-            "status": "PENDING",
-            "duration_seconds": 0,
-            "tests_run": 3,
-            "tests_passed": 0,
-            "tests_failed": 0,
-            "details": []
-        }
-
-        try:
-            # Import and run the test
-            from crawl4ai.agent import test_tools
-
-            # Run the main test function
-            success = await test_tools.main()
-
-            duration = time.time() - start_time
-            suite_result["duration_seconds"] = duration
-
-            if success:
-                suite_result["status"] = "PASS"
-                suite_result["tests_passed"] = 3
-                print(f"\n✓ Tool tests PASSED in {duration:.2f}s")
-            else:
-                suite_result["status"] = "FAIL"
-                suite_result["tests_failed"] = 3
-                print(f"\n✗ Tool tests FAILED in {duration:.2f}s")
-
-        except Exception as e:
-            duration = time.time() - start_time
-            suite_result["status"] = "ERROR"
-            suite_result["error"] = str(e)
-            suite_result["duration_seconds"] = duration
-            suite_result["tests_failed"] = 3
-            print(f"\n✗ Tool tests ERROR: {e}")
-
-        return suite_result
-
-    async def run_scenario_tests(self) -> Dict[str, Any]:
-        """Run multi-turn scenario tests (test_scenarios.py)."""
-        self.print_banner("TEST SUITE 3/3: MULTI-TURN SCENARIO TESTS", "=")
-        print("Testing: 9 scenarios (2 simple, 3 medium, 4 complex)")
-        print("Expected duration: ~3-5 minutes\n")
-
-        start_time = time.time()
-        suite_result = {
-            "name": "Multi-turn Scenario Tests",
-            "file": "test_scenarios.py",
-            "status": "PENDING",
-            "duration_seconds": 0,
-            "tests_run": 9,
-            "tests_passed": 0,
-            "tests_failed": 0,
-            "details": [],
-            "pass_rate_percent": 0.0
-        }
-
-        try:
-            # Import and run the test
-            from crawl4ai.agent import test_scenarios
-
-            # Run all scenarios
-            success = await test_scenarios.run_all_scenarios(self.output_dir)
-
-            duration = time.time() - start_time
-            suite_result["duration_seconds"] = duration
-
-            # Load detailed results from the generated file
-            results_file = self.output_dir / "test_results.json"
-            if results_file.exists():
-                with open(results_file) as f:
-                    scenario_results = json.load(f)
-
-                passed = sum(1 for r in scenario_results if r["status"] == "PASS")
-                total = len(scenario_results)
-
-                suite_result["tests_passed"] = passed
-                suite_result["tests_failed"] = total - passed
-                suite_result["pass_rate_percent"] = (passed / total * 100) if total > 0 else 0
-                suite_result["details"] = scenario_results
-
-                if success:
-                    suite_result["status"] = "PASS"
-                    print(f"\n✓ Scenario tests PASSED ({passed}/{total}) in {duration:.2f}s")
-                else:
-                    suite_result["status"] = "FAIL"
-                    print(f"\n✗ Scenario tests FAILED ({passed}/{total}) in {duration:.2f}s")
-            else:
-                suite_result["status"] = "FAIL"
-                suite_result["tests_failed"] = 9
-                print(f"\n✗ Scenario results file not found")
-
-        except Exception as e:
-            duration = time.time() - start_time
-            suite_result["status"] = "ERROR"
-            suite_result["error"] = str(e)
-            suite_result["duration_seconds"] = duration
-            suite_result["tests_failed"] = 9
-            print(f"\n✗ Scenario tests ERROR: {e}")
-            import traceback
-            traceback.print_exc()
-
-        return suite_result
-
-    async def run_all(self) -> bool:
-        """Run all test suites in sequence."""
-        self.print_banner("CRAWL4AI AGENT - AUTOMATED TEST SUITE", "█")
-        print("This will run 3 test suites in sequence:")
-        print("  1. Component Tests (~5s)")
-        print("  2. Tool Integration Tests (~30s)")
-        print("  3. Multi-turn Scenario Tests (~3-5 min)")
-        print(f"\nOutput directory: {self.output_dir}")
-        print(f"Started at: {self.results['timestamp']}\n")
-
-        overall_start = time.time()
-
-        # Run all test suites
-        component_result = await self.run_component_tests()
-        self.results["test_suites"].append(component_result)
-
-        # Only continue if components pass
-        if component_result["status"] != "PASS":
-            print("\n⚠️  Component tests failed. Stopping execution.")
-            print("Fix component issues before running integration tests.")
-            self.results["overall_status"] = "FAILED"
-            self._save_report()
-            return False
-
-        tool_result = await self.run_tool_tests()
-        self.results["test_suites"].append(tool_result)
-
-        # Only continue if tools pass
-        if tool_result["status"] != "PASS":
-            print("\n⚠️  Tool tests failed. Stopping execution.")
-            print("Fix tool integration issues before running scenarios.")
-            self.results["overall_status"] = "FAILED"
-            self._save_report()
-            return False
-
-        scenario_result = await self.run_scenario_tests()
-        self.results["test_suites"].append(scenario_result)
-
-        # Calculate overall results
-        overall_duration = time.time() - overall_start
-        self.results["total_duration_seconds"] = overall_duration
-
-        # Determine overall status
-        all_passed = all(s["status"] == "PASS" for s in self.results["test_suites"])
-
-        # For scenarios, we accept ≥80% pass rate
-        if scenario_result["status"] == "FAIL" and scenario_result.get("pass_rate_percent", 0) >= 80.0:
-            self.results["overall_status"] = "PASS_WITH_WARNINGS"
-        elif all_passed:
-            self.results["overall_status"] = "PASS"
-        else:
-            self.results["overall_status"] = "FAIL"
-
-        # Print final summary
-        self._print_summary()
-        self._save_report()
-
-        return self.results["overall_status"] in ["PASS", "PASS_WITH_WARNINGS"]
-
-    def _print_summary(self):
-        """Print final test summary."""
-        self.print_banner("FINAL TEST SUMMARY", "█")
-
-        for suite in self.results["test_suites"]:
-            status_icon = "✓" if suite["status"] == "PASS" else "✗"
-            duration = suite["duration_seconds"]
-
-            if "pass_rate_percent" in suite:
-                # Scenario tests
-                passed = suite["tests_passed"]
-                total = suite["tests_run"]
-                pass_rate = suite["pass_rate_percent"]
-                print(f"{status_icon} {suite['name']}: {passed}/{total} passed ({pass_rate:.1f}%) in {duration:.2f}s")
-            else:
-                # Component/Tool tests
-                passed = suite["tests_passed"]
-                total = suite["tests_run"]
-                print(f"{status_icon} {suite['name']}: {passed}/{total} passed in {duration:.2f}s")
-
-        print(f"\nTotal duration: {self.results['total_duration_seconds']:.2f}s")
-        print(f"Overall status: {self.results['overall_status']}")
-
-        if self.results["overall_status"] == "PASS":
-            print("\n🎉 ALL TESTS PASSED! Ready for evaluation phase.")
-        elif self.results["overall_status"] == "PASS_WITH_WARNINGS":
-            print("\n⚠️  Tests passed with warnings (≥80% scenario pass rate).")
-            print("Consider investigating failed scenarios before evaluation.")
-        else:
-            print("\n❌ TESTS FAILED. Please fix issues before proceeding to evaluation.")
-
-    def _save_report(self):
-        """Save detailed test report to JSON."""
-        report_file = self.output_dir / "test_suite_report.json"
-        with open(report_file, "w") as f:
-            json.dump(self.results, f, indent=2)
-
-        print(f"\n📄 Detailed report saved to: {report_file}")
-
-
-async def main():
-    """Main entry point."""
-    # Set up output directory
-    output_dir = Path.cwd() / "test_agent_output"
-
-    # Run all tests
-    runner = TestSuiteRunner(output_dir)
-    success = await runner.run_all()
-
-    return success
-
-
-if __name__ == "__main__":
-    try:
-        success = asyncio.run(main())
-        sys.exit(0 if success else 1)
-    except KeyboardInterrupt:
-        print("\n\n⚠️  Tests interrupted by user")
-        sys.exit(1)
-    except Exception as e:
-        print(f"\n\n❌ Fatal error: {e}")
-        import traceback
-        traceback.print_exc()
-        sys.exit(1)

crawl4ai/agent/terminal_ui.py

@@ -1,289 +0,0 @@
-"""Terminal UI components using Rich for beautiful agent output."""
-
-import readline
-from rich.console import Console
-from rich.markdown import Markdown
-from rich.syntax import Syntax
-from rich.panel import Panel
-from rich.live import Live
-from rich.spinner import Spinner
-from rich.text import Text
-from rich.prompt import Prompt
-from rich.rule import Rule
-
-# Crawl4AI Logo (>X< shape)
-CRAWL4AI_LOGO = """
-  ██      ██
-▓   ██  ██   ▓
- ▓    ██    ▓
-▓   ██  ██   ▓
-  ██      ██
-"""
-
-VERSION = "0.1.0"
-
-
-class TerminalUI:
-    """Rich-based terminal interface for the Crawl4AI agent."""
-
-    def __init__(self):
-        self.console = Console()
-        self._current_text = ""
-
-        # Configure readline for command history
-        # History will persist in memory during session
-        readline.parse_and_bind('tab: complete')  # Enable tab completion
-        readline.parse_and_bind('set editing-mode emacs')  # Emacs-style editing (Ctrl+A, Ctrl+E, etc.)
-        # Up/Down arrows already work by default for history
-
-    def show_header(self, session_id: str = None, log_path: str = None):
-        """Display agent session header - Claude Code style with vertical divider."""
-        import os
-
-        self.console.print()
-
-        # Get current directory
-        current_dir = os.getcwd()
-
-        # Build left and right columns separately to avoid padding issues
-        from rich.table import Table
-        from rich.text import Text
-
-        # Create a table with two columns
-        table = Table.grid(padding=(0, 2))
-        table.add_column(width=30, style="")  # Left column
-        table.add_column(width=1, style="dim")  # Divider
-        table.add_column(style="")  # Right column
-
-        # Row 1: Welcome / Tips header (centered)
-        table.add_row(
-            Text("Welcome back!", style="bold white", justify="center"),
-            "│",
-            Text("Tips", style="bold white")
-        )
-
-        # Row 2: Empty / Tip 1
-        table.add_row(
-            "",
-            "│",
-            Text("• Press ", style="dim") + Text("Enter", style="cyan") + Text(" to send", style="dim")
-        )
-
-        # Row 3: Logo line 1 / Tip 2
-        table.add_row(
-            Text("      ██      ██", style="bold cyan"),
-            "│",
-            Text("• Press ", style="dim") + Text("Option+Enter", style="cyan") + Text(" or ", style="dim") + Text("Ctrl+J", style="cyan") + Text(" for new line", style="dim")
-        )
-
-        # Row 4: Logo line 2 / Tip 3
-        table.add_row(
-            Text("    ▓   ██  ██   ▓", style="bold cyan"),
-            "│",
-            Text("• Use ", style="dim") + Text("/exit", style="cyan") + Text(", ", style="dim") + Text("/clear", style="cyan") + Text(", ", style="dim") + Text("/help", style="cyan") + Text(", ", style="dim") + Text("/browser", style="cyan")
-        )
-
-        # Row 5: Logo line 3 / Empty
-        table.add_row(
-            Text("     ▓    ██    ▓", style="bold cyan"),
-            "│",
-            ""
-        )
-
-        # Row 6: Logo line 4 / Session header
-        table.add_row(
-            Text("    ▓   ██  ██   ▓", style="bold cyan"),
-            "│",
-            Text("Session", style="bold white")
-        )
-
-        # Row 7: Logo line 5 / Session ID
-        session_name = os.path.basename(session_id) if session_id else "unknown"
-        table.add_row(
-            Text("      ██      ██", style="bold cyan"),
-            "│",
-            Text(session_name, style="dim")
-        )
-
-        # Row 8: Empty
-        table.add_row("", "│", "")
-
-        # Row 9: Version (centered)
-        table.add_row(
-            Text(f"Version {VERSION}", style="dim", justify="center"),
-            "│",
-            ""
-        )
-
-        # Row 10: Path (centered)
-        table.add_row(
-            Text(current_dir, style="dim", justify="center"),
-            "│",
-            ""
-        )
-
-        # Create panel with title
-        panel = Panel(
-            table,
-            title=f"[bold cyan]─── Crawl4AI Agent v{VERSION} ───[/bold cyan]",
-            title_align="left",
-            border_style="cyan",
-            padding=(1, 1),
-            expand=True
-        )
-
-        self.console.print(panel)
-        self.console.print()
-
-    def show_commands(self):
-        """Display available commands."""
-        self.console.print("\n[dim]Commands:[/dim]")
-        self.console.print("  [cyan]/exit[/cyan] - Exit chat")
-        self.console.print("  [cyan]/clear[/cyan] - Clear screen")
-        self.console.print("  [cyan]/help[/cyan] - Show this help")
-        self.console.print("  [cyan]/browser[/cyan] - Show browser status\n")
-
-    def get_user_input(self) -> str:
-        """Get user input with multi-line support and paste handling.
-
-        Usage:
-        - Press Enter to submit
-        - Press Option+Enter (or Ctrl+J) for new line
-        - Paste multi-line text works perfectly
-        """
-        from prompt_toolkit import prompt
-        from prompt_toolkit.key_binding import KeyBindings
-        from prompt_toolkit.keys import Keys
-        from prompt_toolkit.formatted_text import HTML
-
-        # Create custom key bindings
-        bindings = KeyBindings()
-
-        # Enter to submit (reversed from default multiline behavior)
-        @bindings.add(Keys.Enter)
-        def _(event):
-            """Submit the input when Enter is pressed."""
-            event.current_buffer.validate_and_handle()
-
-        # Option+Enter for newline (sends Esc+Enter when iTerm2 configured with "Esc+")
-        @bindings.add(Keys.Escape, Keys.Enter)
-        def _(event):
-            """Insert newline with Option+Enter (or Esc then Enter)."""
-            event.current_buffer.insert_text("\n")
-
-        # Ctrl+J as alternative for newline (works everywhere)
-        @bindings.add(Keys.ControlJ)
-        def _(event):
-            """Insert newline with Ctrl+J."""
-            event.current_buffer.insert_text("\n")
-
-        try:
-            # Tips are now in header, no need for extra hint
-
-            # Use prompt_toolkit with HTML formatting (no ANSI codes)
-            user_input = prompt(
-                HTML("\n<ansigreen><b>You:</b></ansigreen> "),
-                multiline=True,
-                key_bindings=bindings,
-                enable_open_in_editor=False,
-            )
-            return user_input.strip()
-
-        except (EOFError, KeyboardInterrupt):
-            raise EOFError()
-
-    def print_separator(self):
-        """Print a visual separator."""
-        self.console.print(Rule(style="dim"))
-
-    def print_thinking(self):
-        """Show thinking indicator."""
-        self.console.print("\n[cyan]Agent:[/cyan] [dim]thinking...[/dim]", end="")
-
-    def print_agent_text(self, text: str, stream: bool = False):
-        """
-        Print agent response text.
-
-        Args:
-            text: Text to print
-            stream: If True, append to current streaming output
-        """
-        if stream:
-            # For streaming, just print without newline
-            self.console.print(f"\r[cyan]Agent:[/cyan] {text}", end="")
-        else:
-            # For complete messages
-            self.console.print(f"\n[cyan]Agent:[/cyan] {text}")
-
-    def print_markdown(self, markdown_text: str):
-        """Render markdown content."""
-        self.console.print()
-        self.console.print(Markdown(markdown_text))
-
-    def print_code(self, code: str, language: str = "python"):
-        """Render code with syntax highlighting."""
-        self.console.print()
-        self.console.print(Syntax(code, language, theme="monokai", line_numbers=True))
-
-    def print_error(self, error_msg: str):
-        """Display error message."""
-        self.console.print(f"\n[bold red]Error:[/bold red] {error_msg}")
-
-    def print_success(self, msg: str):
-        """Display success message."""
-        self.console.print(f"\n[bold green]✓[/bold green] {msg}")
-
-    def print_info(self, msg: str):
-        """Display info message."""
-        self.console.print(f"\n[bold blue]ℹ[/bold blue] {msg}")
-
-    def clear_screen(self):
-        """Clear the terminal screen."""
-        self.console.clear()
-
-    def print_session_summary(self, duration_s: float, turns: int, cost_usd: float = None):
-        """Display session completion summary."""
-        self.console.print()
-        self.console.print(Panel(
-            f"[green]✅ Completed[/green]\n"
-            f"⏱ Duration: {duration_s:.2f}s\n"
-            f"🔄 Turns: {turns}\n"
-            + (f"💰 Cost: ${cost_usd:.4f}" if cost_usd else ""),
-            border_style="green"
-        ))
-
-    def print_tool_use(self, tool_name: str, tool_input: dict = None):
-        """Indicate tool usage with parameters."""
-        # Shorten crawl4ai tool names for readability
-        display_name = tool_name.replace("mcp__crawler__", "")
-
-        if tool_input:
-            # Show key parameters only
-            params = []
-            if "url" in tool_input:
-                url = tool_input["url"]
-                # Truncate long URLs
-                if len(url) > 50:
-                    url = url[:47] + "..."
-                params.append(f"[dim]url=[/dim]{url}")
-            if "session_id" in tool_input:
-                params.append(f"[dim]session=[/dim]{tool_input['session_id']}")
-            if "file_path" in tool_input:
-                params.append(f"[dim]file=[/dim]{tool_input['file_path']}")
-            if "output_format" in tool_input:
-                params.append(f"[dim]format=[/dim]{tool_input['output_format']}")
-
-            param_str = ", ".join(params) if params else ""
-            self.console.print(f"  [yellow]🔧 {display_name}[/yellow]({param_str})")
-        else:
-            self.console.print(f"  [yellow]🔧 {display_name}[/yellow]")
-
-    def with_spinner(self, text: str = "Processing..."):
-        """
-        Context manager for showing a spinner.
-
-        Usage:
-            with ui.with_spinner("Crawling page..."):
-                # do work
-        """
-        return self.console.status(f"[cyan]{text}[/cyan]", spinner="dots")

crawl4ai/agent/test_chat.py

@@ -1,114 +0,0 @@
-#!/usr/bin/env python
-"""Test script to verify chat mode setup (non-interactive)."""
-
-import sys
-import asyncio
-from pathlib import Path
-
-# Add parent to path for imports
-sys.path.insert(0, str(Path(__file__).parent.parent.parent))
-
-from crawl4ai.agent.browser_manager import BrowserManager
-from crawl4ai.agent.terminal_ui import TerminalUI
-from crawl4ai.agent.chat_mode import ChatMode
-from crawl4ai.agent.c4ai_tools import CRAWL_TOOLS
-from crawl4ai.agent.c4ai_prompts import SYSTEM_PROMPT
-
-from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server
-
-
-class MockStorage:
-    """Mock storage for testing."""
-
-    def log(self, event_type: str, data: dict):
-        print(f"[LOG] {event_type}: {data}")
-
-    def get_session_path(self):
-        return "/tmp/test_session.jsonl"
-
-
-async def test_components():
-    """Test individual components."""
-
-    print("="*60)
-    print("CHAT MODE COMPONENT TESTS")
-    print("="*60)
-
-    # Test 1: BrowserManager
-    print("\n[TEST 1] BrowserManager singleton")
-    try:
-        browser1 = await BrowserManager.get_browser()
-        browser2 = await BrowserManager.get_browser()
-        assert browser1 is browser2, "Browser instances should be same (singleton)"
-        print("✓ BrowserManager singleton works")
-        await BrowserManager.close_browser()
-    except Exception as e:
-        print(f"✗ BrowserManager failed: {e}")
-        return False
-
-    # Test 2: TerminalUI
-    print("\n[TEST 2] TerminalUI rendering")
-    try:
-        ui = TerminalUI()
-        ui.show_header("test-123", "/tmp/test.log")
-        ui.print_agent_text("Hello from agent")
-        ui.print_markdown("# Test\nThis is **bold**")
-        ui.print_success("Test success message")
-        print("✓ TerminalUI renders correctly")
-    except Exception as e:
-        print(f"✗ TerminalUI failed: {e}")
-        return False
-
-    # Test 3: MCP Server Setup
-    print("\n[TEST 3] MCP Server with tools")
-    try:
-        crawler_server = create_sdk_mcp_server(
-            name="crawl4ai",
-            version="1.0.0",
-            tools=CRAWL_TOOLS
-        )
-        print(f"✓ MCP server created with {len(CRAWL_TOOLS)} tools")
-    except Exception as e:
-        print(f"✗ MCP Server failed: {e}")
-        return False
-
-    # Test 4: ChatMode instantiation
-    print("\n[TEST 4] ChatMode instantiation")
-    try:
-        options = ClaudeAgentOptions(
-            mcp_servers={"crawler": crawler_server},
-            allowed_tools=[
-                "mcp__crawler__quick_crawl",
-                "mcp__crawler__start_session",
-                "mcp__crawler__navigate",
-                "mcp__crawler__extract_data",
-                "mcp__crawler__execute_js",
-                "mcp__crawler__screenshot",
-                "mcp__crawler__close_session",
-            ],
-            system_prompt=SYSTEM_PROMPT,
-            permission_mode="acceptEdits"
-        )
-
-        ui = TerminalUI()
-        storage = MockStorage()
-        chat = ChatMode(options, ui, storage)
-        print("✓ ChatMode instance created successfully")
-    except Exception as e:
-        print(f"✗ ChatMode failed: {e}")
-        import traceback
-        traceback.print_exc()
-        return False
-
-    print("\n" + "="*60)
-    print("ALL COMPONENT TESTS PASSED ✓")
-    print("="*60)
-    print("\nTo test interactive chat mode, run:")
-    print("  python -m crawl4ai.agent.agent_crawl --chat")
-
-    return True
-
-
-if __name__ == "__main__":
-    success = asyncio.run(test_components())
-    sys.exit(0 if success else 1)

crawl4ai/agent/test_scenarios.py

@@ -1,524 +0,0 @@
-#!/usr/bin/env python
-"""
-Automated multi-turn chat scenario tests for Crawl4AI Agent.
-
-Tests agent's ability to handle complex conversations, maintain state,
-plan and execute tasks without human interaction.
-"""
-
-import asyncio
-import json
-import time
-from pathlib import Path
-from typing import List, Dict, Any, Optional
-from dataclasses import dataclass
-from enum import Enum
-
-from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions, create_sdk_mcp_server
-from claude_agent_sdk import AssistantMessage, TextBlock, ResultMessage, ToolUseBlock
-
-from .c4ai_tools import CRAWL_TOOLS
-from .c4ai_prompts import SYSTEM_PROMPT
-from .browser_manager import BrowserManager
-
-
-class TurnResult(Enum):
-    """Result of a single conversation turn."""
-    PASS = "PASS"
-    FAIL = "FAIL"
-    TIMEOUT = "TIMEOUT"
-    ERROR = "ERROR"
-
-
-@dataclass
-class TurnExpectation:
-    """Expectations for a single conversation turn."""
-    user_message: str
-    expect_tools: Optional[List[str]] = None  # Tools that should be called
-    expect_keywords: Optional[List[str]] = None  # Keywords in response
-    expect_files_created: Optional[List[str]] = None  # File patterns created
-    expect_success: bool = True  # Should complete without error
-    expect_min_turns: int = 1  # Minimum agent turns to complete
-    timeout_seconds: int = 60
-
-
-@dataclass
-class Scenario:
-    """A complete multi-turn conversation scenario."""
-    name: str
-    category: str  # "simple", "medium", "complex"
-    description: str
-    turns: List[TurnExpectation]
-    cleanup_files: Optional[List[str]] = None  # Files to cleanup after test
-
-
-# =============================================================================
-# TEST SCENARIOS - Categorized from Simple to Complex
-# =============================================================================
-
-SIMPLE_SCENARIOS = [
-    Scenario(
-        name="Single quick crawl",
-        category="simple",
-        description="Basic one-shot crawl with markdown extraction",
-        turns=[
-            TurnExpectation(
-                user_message="Use quick_crawl to get the title from example.com",
-                expect_tools=["mcp__crawler__quick_crawl"],
-                expect_keywords=["Example Domain", "title"],
-                timeout_seconds=30
-            )
-        ]
-    ),
-
-    Scenario(
-        name="Session lifecycle",
-        category="simple",
-        description="Start session, navigate, close - basic session management",
-        turns=[
-            TurnExpectation(
-                user_message="Start a session named 'simple_test'",
-                expect_tools=["mcp__crawler__start_session"],
-                expect_keywords=["session", "started"],
-                timeout_seconds=20
-            ),
-            TurnExpectation(
-                user_message="Navigate to example.com",
-                expect_tools=["mcp__crawler__navigate"],
-                expect_keywords=["navigated", "example.com"],
-                timeout_seconds=25
-            ),
-            TurnExpectation(
-                user_message="Close the session",
-                expect_tools=["mcp__crawler__close_session"],
-                expect_keywords=["closed"],
-                timeout_seconds=15
-            )
-        ]
-    ),
-]
-
-
-MEDIUM_SCENARIOS = [
-    Scenario(
-        name="Multi-page crawl with file output",
-        category="medium",
-        description="Crawl multiple pages and save results to file",
-        turns=[
-            TurnExpectation(
-                user_message="Crawl example.com and example.org, extract titles from both",
-                expect_tools=["mcp__crawler__quick_crawl"],
-                expect_min_turns=2,  # Should make 2 separate crawls
-                timeout_seconds=45
-            ),
-            TurnExpectation(
-                user_message="Use the Write tool to save the titles you extracted to a file called crawl_results.txt",
-                expect_tools=["Write"],
-                expect_files_created=["crawl_results.txt"],
-                timeout_seconds=30
-            )
-        ],
-        cleanup_files=["crawl_results.txt"]
-    ),
-
-    Scenario(
-        name="Session-based data extraction",
-        category="medium",
-        description="Use session to navigate and extract data in steps",
-        turns=[
-            TurnExpectation(
-                user_message="Start session 'extract_test', navigate to example.com, and extract the markdown",
-                expect_tools=["mcp__crawler__start_session", "mcp__crawler__navigate", "mcp__crawler__extract_data"],
-                expect_keywords=["Example Domain"],
-                timeout_seconds=50
-            ),
-            TurnExpectation(
-                user_message="Use the Write tool to save the extracted markdown to example_content.md",
-                expect_tools=["Write"],
-                expect_files_created=["example_content.md"],
-                timeout_seconds=30
-            ),
-            TurnExpectation(
-                user_message="Close the session",
-                expect_tools=["mcp__crawler__close_session"],
-                timeout_seconds=15
-            )
-        ],
-        cleanup_files=["example_content.md"]
-    ),
-
-    Scenario(
-        name="Context retention across turns",
-        category="medium",
-        description="Agent should remember previous context",
-        turns=[
-            TurnExpectation(
-                user_message="Crawl example.com and tell me the title",
-                expect_tools=["mcp__crawler__quick_crawl"],
-                expect_keywords=["Example Domain"],
-                timeout_seconds=30
-            ),
-            TurnExpectation(
-                user_message="What was the URL I just asked you to crawl?",
-                expect_keywords=["example.com"],
-                expect_tools=[],  # Should answer from memory, no tools needed
-                timeout_seconds=15
-            )
-        ]
-    ),
-]
-
-
-COMPLEX_SCENARIOS = [
-    Scenario(
-        name="Multi-step task with planning",
-        category="complex",
-        description="Complex task requiring agent to plan, execute, and verify",
-        turns=[
-            TurnExpectation(
-                user_message="Crawl example.com and example.org, compare their content, and create a markdown report with: 1) titles of both, 2) word count comparison, 3) save to comparison_report.md",
-                expect_tools=["mcp__crawler__quick_crawl", "Write"],
-                expect_files_created=["comparison_report.md"],
-                expect_min_turns=3,  # Plan, crawl both, write report
-                timeout_seconds=90
-            ),
-            TurnExpectation(
-                user_message="Read back the report you just created",
-                expect_tools=["Read"],
-                expect_keywords=["Example Domain"],
-                timeout_seconds=20
-            )
-        ],
-        cleanup_files=["comparison_report.md"]
-    ),
-
-    Scenario(
-        name="Session with state manipulation",
-        category="complex",
-        description="Complex session workflow with multiple operations",
-        turns=[
-            TurnExpectation(
-                user_message="Start session 'complex_session' and navigate to example.com",
-                expect_tools=["mcp__crawler__start_session", "mcp__crawler__navigate"],
-                timeout_seconds=30
-            ),
-            TurnExpectation(
-                user_message="Extract the page content and count how many times the word 'example' appears (case insensitive)",
-                expect_tools=["mcp__crawler__extract_data"],
-                expect_keywords=["example"],
-                timeout_seconds=30
-            ),
-            TurnExpectation(
-                user_message="Take a screenshot of the current page",
-                expect_tools=["mcp__crawler__screenshot"],
-                expect_keywords=["screenshot"],
-                timeout_seconds=25
-            ),
-            TurnExpectation(
-                user_message="Close the session",
-                expect_tools=["mcp__crawler__close_session"],
-                timeout_seconds=15
-            )
-        ]
-    ),
-
-    Scenario(
-        name="Error recovery and continuation",
-        category="complex",
-        description="Agent should handle errors gracefully and continue",
-        turns=[
-            TurnExpectation(
-                user_message="Crawl https://this-site-definitely-does-not-exist-12345.com",
-                expect_success=False,  # Should fail gracefully
-                expect_keywords=["error", "fail"],
-                timeout_seconds=30
-            ),
-            TurnExpectation(
-                user_message="That's okay, crawl example.com instead",
-                expect_tools=["mcp__crawler__quick_crawl"],
-                expect_keywords=["Example Domain"],
-                timeout_seconds=30
-            )
-        ]
-    ),
-]
-
-
-# Combine all scenarios
-ALL_SCENARIOS = SIMPLE_SCENARIOS + MEDIUM_SCENARIOS + COMPLEX_SCENARIOS
-
-
-# =============================================================================
-# TEST RUNNER
-# =============================================================================
-
-class ScenarioRunner:
-    """Runs automated chat scenarios without human interaction."""
-
-    def __init__(self, working_dir: Path):
-        self.working_dir = working_dir
-        self.results = []
-
-    async def run_scenario(self, scenario: Scenario) -> Dict[str, Any]:
-        """Run a single scenario and return results."""
-        print(f"\n{'='*70}")
-        print(f"[{scenario.category.upper()}] {scenario.name}")
-        print(f"{'='*70}")
-        print(f"Description: {scenario.description}\n")
-
-        start_time = time.time()
-        turn_results = []
-
-        try:
-            # Setup agent options
-            crawler_server = create_sdk_mcp_server(
-                name="crawl4ai",
-                version="1.0.0",
-                tools=CRAWL_TOOLS
-            )
-
-            options = ClaudeAgentOptions(
-                mcp_servers={"crawler": crawler_server},
-                allowed_tools=[
-                    "mcp__crawler__quick_crawl",
-                    "mcp__crawler__start_session",
-                    "mcp__crawler__navigate",
-                    "mcp__crawler__extract_data",
-                    "mcp__crawler__execute_js",
-                    "mcp__crawler__screenshot",
-                    "mcp__crawler__close_session",
-                    "Read", "Write", "Edit", "Glob", "Grep", "Bash"
-                ],
-                system_prompt=SYSTEM_PROMPT,
-                permission_mode="acceptEdits",
-                cwd=str(self.working_dir)
-            )
-
-            # Run conversation
-            async with ClaudeSDKClient(options=options) as client:
-                for turn_idx, expectation in enumerate(scenario.turns, 1):
-                    print(f"\nTurn {turn_idx}: {expectation.user_message}")
-
-                    turn_result = await self._run_turn(
-                        client, expectation, turn_idx
-                    )
-                    turn_results.append(turn_result)
-
-                    if turn_result["status"] != TurnResult.PASS.value:
-                        print(f"  ✗ FAILED: {turn_result['reason']}")
-                        break
-                    else:
-                        print(f"  ✓ PASSED")
-
-            # Cleanup
-            if scenario.cleanup_files:
-                self._cleanup_files(scenario.cleanup_files)
-
-            # Overall result
-            all_passed = all(r["status"] == TurnResult.PASS.value for r in turn_results)
-            duration = time.time() - start_time
-
-            result = {
-                "scenario": scenario.name,
-                "category": scenario.category,
-                "status": "PASS" if all_passed else "FAIL",
-                "duration_seconds": duration,
-                "turns": turn_results
-            }
-
-            return result
-
-        except Exception as e:
-            print(f"\n✗ SCENARIO ERROR: {e}")
-            return {
-                "scenario": scenario.name,
-                "category": scenario.category,
-                "status": "ERROR",
-                "error": str(e),
-                "duration_seconds": time.time() - start_time,
-                "turns": turn_results
-            }
-        finally:
-            # Ensure browser cleanup
-            await BrowserManager.close_browser()
-
-    async def _run_turn(
-        self,
-        client: ClaudeSDKClient,
-        expectation: TurnExpectation,
-        turn_number: int
-    ) -> Dict[str, Any]:
-        """Execute a single conversation turn and validate."""
-
-        tools_used = []
-        response_text = ""
-        agent_turns = 0
-
-        try:
-            # Send user message
-            await client.query(expectation.user_message)
-
-            # Collect response
-            start_time = time.time()
-            async for message in client.receive_messages():
-                if time.time() - start_time > expectation.timeout_seconds:
-                    return {
-                        "turn": turn_number,
-                        "status": TurnResult.TIMEOUT.value,
-                        "reason": f"Exceeded {expectation.timeout_seconds}s timeout"
-                    }
-
-                if isinstance(message, AssistantMessage):
-                    agent_turns += 1
-                    for block in message.content:
-                        if isinstance(block, TextBlock):
-                            response_text += block.text + " "
-                        elif isinstance(block, ToolUseBlock):
-                            tools_used.append(block.name)
-
-                elif isinstance(message, ResultMessage):
-                    # Check if error when expecting success
-                    if expectation.expect_success and message.is_error:
-                        return {
-                            "turn": turn_number,
-                            "status": TurnResult.FAIL.value,
-                            "reason": f"Agent returned error: {message.result}"
-                        }
-                    break
-
-            # Validate expectations
-            validation = self._validate_turn(
-                expectation, tools_used, response_text, agent_turns
-            )
-
-            return {
-                "turn": turn_number,
-                "status": validation["status"],
-                "reason": validation.get("reason", "All checks passed"),
-                "tools_used": tools_used,
-                "agent_turns": agent_turns
-            }
-
-        except Exception as e:
-            return {
-                "turn": turn_number,
-                "status": TurnResult.ERROR.value,
-                "reason": f"Exception: {str(e)}"
-            }
-
-    def _validate_turn(
-        self,
-        expectation: TurnExpectation,
-        tools_used: List[str],
-        response_text: str,
-        agent_turns: int
-    ) -> Dict[str, Any]:
-        """Validate turn results against expectations."""
-
-        # Check expected tools
-        if expectation.expect_tools:
-            for tool in expectation.expect_tools:
-                if tool not in tools_used:
-                    return {
-                        "status": TurnResult.FAIL.value,
-                        "reason": f"Expected tool '{tool}' was not used"
-                    }
-
-        # Check keywords
-        if expectation.expect_keywords:
-            response_lower = response_text.lower()
-            for keyword in expectation.expect_keywords:
-                if keyword.lower() not in response_lower:
-                    return {
-                        "status": TurnResult.FAIL.value,
-                        "reason": f"Expected keyword '{keyword}' not found in response"
-                    }
-
-        # Check files created
-        if expectation.expect_files_created:
-            for pattern in expectation.expect_files_created:
-                matches = list(self.working_dir.glob(pattern))
-                if not matches:
-                    return {
-                        "status": TurnResult.FAIL.value,
-                        "reason": f"Expected file matching '{pattern}' was not created"
-                    }
-
-        # Check minimum turns
-        if agent_turns < expectation.expect_min_turns:
-            return {
-                "status": TurnResult.FAIL.value,
-                "reason": f"Expected at least {expectation.expect_min_turns} agent turns, got {agent_turns}"
-            }
-
-        return {"status": TurnResult.PASS.value}
-
-    def _cleanup_files(self, patterns: List[str]):
-        """Remove files created during test."""
-        for pattern in patterns:
-            for file_path in self.working_dir.glob(pattern):
-                try:
-                    file_path.unlink()
-                except Exception as e:
-                    print(f"  Warning: Could not delete {file_path}: {e}")
-
-
-async def run_all_scenarios(working_dir: Optional[Path] = None):
-    """Run all test scenarios and report results."""
-
-    if working_dir is None:
-        working_dir = Path.cwd() / "test_agent_output"
-        working_dir.mkdir(exist_ok=True)
-
-    runner = ScenarioRunner(working_dir)
-
-    print("\n" + "="*70)
-    print("CRAWL4AI AGENT SCENARIO TESTS")
-    print("="*70)
-    print(f"Working directory: {working_dir}")
-    print(f"Total scenarios: {len(ALL_SCENARIOS)}")
-    print(f"  Simple: {len(SIMPLE_SCENARIOS)}")
-    print(f"  Medium: {len(MEDIUM_SCENARIOS)}")
-    print(f"  Complex: {len(COMPLEX_SCENARIOS)}")
-
-    results = []
-    for scenario in ALL_SCENARIOS:
-        result = await runner.run_scenario(scenario)
-        results.append(result)
-
-    # Summary
-    print("\n" + "="*70)
-    print("TEST SUMMARY")
-    print("="*70)
-
-    by_category = {"simple": [], "medium": [], "complex": []}
-    for result in results:
-        by_category[result["category"]].append(result)
-
-    for category in ["simple", "medium", "complex"]:
-        cat_results = by_category[category]
-        passed = sum(1 for r in cat_results if r["status"] == "PASS")
-        total = len(cat_results)
-        print(f"\n{category.upper()}: {passed}/{total} passed")
-        for r in cat_results:
-            status_icon = "✓" if r["status"] == "PASS" else "✗"
-            print(f"  {status_icon} {r['scenario']} ({r['duration_seconds']:.1f}s)")
-
-    total_passed = sum(1 for r in results if r["status"] == "PASS")
-    total = len(results)
-
-    print(f"\nOVERALL: {total_passed}/{total} scenarios passed ({total_passed/total*100:.1f}%)")
-
-    # Save detailed results
-    results_file = working_dir / "test_results.json"
-    with open(results_file, "w") as f:
-        json.dump(results, f, indent=2)
-    print(f"\nDetailed results saved to: {results_file}")
-
-    return total_passed == total
-
-
-if __name__ == "__main__":
-    import sys
-    success = asyncio.run(run_all_scenarios())
-    sys.exit(0 if success else 1)

crawl4ai/agent/test_tools.py

@@ -1,140 +0,0 @@
-#!/usr/bin/env python
-"""Test script for Crawl4AI tools - tests tools directly without the agent."""
-
-import asyncio
-import json
-from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
-
-async def test_quick_crawl():
-    """Test quick_crawl tool logic directly."""
-    print("\n" + "="*60)
-    print("TEST 1: Quick Crawl - Markdown Format")
-    print("="*60)
-
-    crawler_config = BrowserConfig(headless=True, verbose=False)
-    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
-
-    async with AsyncWebCrawler(config=crawler_config) as crawler:
-        result = await crawler.arun(url="https://example.com", config=run_config)
-
-        print(f"Success: {result.success}")
-        print(f"URL: {result.url}")
-
-        # Handle markdown - can be string or MarkdownGenerationResult object
-        if isinstance(result.markdown, str):
-            markdown_content = result.markdown
-        elif hasattr(result.markdown, 'raw_markdown'):
-            markdown_content = result.markdown.raw_markdown
-        else:
-            markdown_content = str(result.markdown)
-
-        print(f"Markdown type: {type(result.markdown)}")
-        print(f"Markdown length: {len(markdown_content)}")
-        print(f"Markdown preview:\n{markdown_content[:300]}")
-
-        return result.success
-
-
-async def test_session_workflow():
-    """Test session-based workflow."""
-    print("\n" + "="*60)
-    print("TEST 2: Session-Based Workflow")
-    print("="*60)
-
-    crawler_config = BrowserConfig(headless=True, verbose=False)
-
-    # Start session
-    crawler = AsyncWebCrawler(config=crawler_config)
-    await crawler.__aenter__()
-    print("✓ Session started")
-
-    try:
-        # Navigate to URL
-        run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
-        result = await crawler.arun(url="https://example.com", config=run_config)
-        print(f"✓ Navigated to {result.url}, success: {result.success}")
-
-        # Extract data
-        if isinstance(result.markdown, str):
-            markdown_content = result.markdown
-        elif hasattr(result.markdown, 'raw_markdown'):
-            markdown_content = result.markdown.raw_markdown
-        else:
-            markdown_content = str(result.markdown)
-
-        print(f"✓ Extracted {len(markdown_content)} chars of markdown")
-        print(f"  Preview: {markdown_content[:200]}")
-
-        # Screenshot test - need to re-fetch with screenshot enabled
-        screenshot_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, screenshot=True)
-        result2 = await crawler.arun(url=result.url, config=screenshot_config)
-        print(f"✓ Screenshot captured: {result2.screenshot is not None}")
-
-        return True
-
-    finally:
-        # Close session
-        await crawler.__aexit__(None, None, None)
-        print("✓ Session closed")
-
-
-async def test_html_format():
-    """Test HTML output format."""
-    print("\n" + "="*60)
-    print("TEST 3: Quick Crawl - HTML Format")
-    print("="*60)
-
-    crawler_config = BrowserConfig(headless=True, verbose=False)
-    run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
-
-    async with AsyncWebCrawler(config=crawler_config) as crawler:
-        result = await crawler.arun(url="https://example.com", config=run_config)
-
-        print(f"Success: {result.success}")
-        print(f"HTML length: {len(result.html)}")
-        print(f"HTML preview:\n{result.html[:300]}")
-
-        return result.success
-
-
-async def main():
-    """Run all tests."""
-    print("\n" + "="*70)
-    print(" CRAWL4AI TOOLS TEST SUITE")
-    print("="*70)
-
-    tests = [
-        ("Quick Crawl (Markdown)", test_quick_crawl),
-        ("Session Workflow", test_session_workflow),
-        ("Quick Crawl (HTML)", test_html_format),
-    ]
-
-    results = []
-    for name, test_func in tests:
-        try:
-            result = await test_func()
-            results.append((name, result, None))
-        except Exception as e:
-            results.append((name, False, str(e)))
-
-    # Summary
-    print("\n" + "="*70)
-    print(" TEST SUMMARY")
-    print("="*70)
-
-    for name, success, error in results:
-        status = "✓ PASS" if success else "✗ FAIL"
-        print(f"{status} - {name}")
-        if error:
-            print(f"     Error: {error}")
-
-    total = len(results)
-    passed = sum(1 for _, success, _ in results if success)
-    print(f"\nTotal: {total} | Passed: {passed} | Failed: {total - passed}")
-
-    return all(success for _, success, _ in results)
-
-
-if __name__ == "__main__":
-    success = asyncio.run(main())
-    exit(0 if success else 1)

crawl4ai/async_database.py

@@ -2,233 +2,19 @@ import os
 from pathlib import Path
 import aiosqlite
 import asyncio
-from typing import Optional, Dict
-from contextlib import asynccontextmanager
-import json  
-from .models import CrawlResult, MarkdownGenerationResult, StringCompatibleMarkdown
-import aiofiles
-from .async_logger import AsyncLogger
+from typing import Optional, Tuple
 
-from .utils import ensure_content_dirs, generate_content_hash
-from .utils import VersionManager
-from .utils import get_error_context, create_box_message
-
-base_directory = DB_PATH = os.path.join(
-    os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai"
-)
+DB_PATH = os.path.join(Path.home(), ".crawl4ai")
 os.makedirs(DB_PATH, exist_ok=True)
-DB_PATH = os.path.join(base_directory, "crawl4ai.db")
-
+DB_PATH = os.path.join(DB_PATH, "crawl4ai.db")
 
 class AsyncDatabaseManager:
-    def __init__(self, pool_size: int = 10, max_retries: int = 3):
+    def __init__(self):
         self.db_path = DB_PATH
-        self.content_paths = ensure_content_dirs(os.path.dirname(DB_PATH))
-        self.pool_size = pool_size
-        self.max_retries = max_retries
-        self.connection_pool: Dict[int, aiosqlite.Connection] = {}
-        self.pool_lock = asyncio.Lock()
-        self.init_lock = asyncio.Lock()
-        self.connection_semaphore = asyncio.Semaphore(pool_size)
-        self._initialized = False
-        self.version_manager = VersionManager()
-        self.logger = AsyncLogger(
-            log_file=os.path.join(base_directory, ".crawl4ai", "crawler_db.log"),
-            verbose=False,
-            tag_width=10,
-        )
-
-    async def initialize(self):
-        """Initialize the database and connection pool"""
-        try:
-            self.logger.info("Initializing database", tag="INIT")
-            # Ensure the database file exists
-            os.makedirs(os.path.dirname(self.db_path), exist_ok=True)
-
-            # Check if version update is needed
-            needs_update = self.version_manager.needs_update()
-
-            # Always ensure base table exists
-            await self.ainit_db()
-
-            # Verify the table exists
-            async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
-                async with db.execute(
-                    "SELECT name FROM sqlite_master WHERE type='table' AND name='crawled_data'"
-                ) as cursor:
-                    result = await cursor.fetchone()
-                    if not result:
-                        raise Exception("crawled_data table was not created")
-
-            # If version changed or fresh install, run updates
-            if needs_update:
-                self.logger.info("New version detected, running updates", tag="INIT")
-                await self.update_db_schema()
-                from .migrations import (
-                    run_migration,
-                )  # Import here to avoid circular imports
-
-                await run_migration()
-                self.version_manager.update_version()  # Update stored version after successful migration
-                self.logger.success(
-                    "Version update completed successfully", tag="COMPLETE"
-                )
-            else:
-                self.logger.success(
-                    "Database initialization completed successfully", tag="COMPLETE"
-                )
-
-        except Exception as e:
-            self.logger.error(
-                message="Database initialization error: {error}",
-                tag="ERROR",
-                params={"error": str(e)},
-            )
-            self.logger.info(
-                message="Database will be initialized on first use", tag="INIT"
-            )
-
-            raise
-
-    async def cleanup(self):
-        """Cleanup connections when shutting down"""
-        async with self.pool_lock:
-            for conn in self.connection_pool.values():
-                await conn.close()
-            self.connection_pool.clear()
-
-    @asynccontextmanager
-    async def get_connection(self):
-        """Connection pool manager with enhanced error handling"""
-        if not self._initialized:
-            async with self.init_lock:
-                if not self._initialized:
-                    try:
-                        await self.initialize()
-                        self._initialized = True
-                    except Exception as e:
-                        import sys
-
-                        error_context = get_error_context(sys.exc_info())
-                        self.logger.error(
-                            message="Database initialization failed:\n{error}\n\nContext:\n{context}\n\nTraceback:\n{traceback}",
-                            tag="ERROR",
-                            force_verbose=True,
-                            params={
-                                "error": str(e),
-                                "context": error_context["code_context"],
-                                "traceback": error_context["full_traceback"],
-                            },
-                        )
-                        raise
-
-        await self.connection_semaphore.acquire()
-        task_id = id(asyncio.current_task())
-
-        try:
-            async with self.pool_lock:
-                if task_id not in self.connection_pool:
-                    try:
-                        conn = await aiosqlite.connect(self.db_path, timeout=30.0)
-                        await conn.execute("PRAGMA journal_mode = WAL")
-                        await conn.execute("PRAGMA busy_timeout = 5000")
-
-                        # Verify database structure
-                        async with conn.execute(
-                            "PRAGMA table_info(crawled_data)"
-                        ) as cursor:
-                            columns = await cursor.fetchall()
-                            column_names = [col[1] for col in columns]
-                            expected_columns = {
-                                "url",
-                                "html",
-                                "cleaned_html",
-                                "markdown",
-                                "extracted_content",
-                                "success",
-                                "media",
-                                "links",
-                                "metadata",
-                                "screenshot",
-                                "response_headers",
-                                "downloaded_files",
-                            }
-                            missing_columns = expected_columns - set(column_names)
-                            if missing_columns:
-                                raise ValueError(
-                                    f"Database missing columns: {missing_columns}"
-                                )
-
-                        self.connection_pool[task_id] = conn
-                    except Exception as e:
-                        import sys
-
-                        error_context = get_error_context(sys.exc_info())
-                        error_message = (
-                            f"Unexpected error in db get_connection at line {error_context['line_no']} "
-                            f"in {error_context['function']} ({error_context['filename']}):\n"
-                            f"Error: {str(e)}\n\n"
-                            f"Code context:\n{error_context['code_context']}"
-                        )
-                        self.logger.error(
-                            message="{error}",
-                            tag="ERROR",
-                            params={"error": str(error_message)},
-                            boxes=["error"],
-                        )
-
-                        raise
-
-            yield self.connection_pool[task_id]
-
-        except Exception as e:
-            import sys
-
-            error_context = get_error_context(sys.exc_info())
-            error_message = (
-                f"Unexpected error in db get_connection at line {error_context['line_no']} "
-                f"in {error_context['function']} ({error_context['filename']}):\n"
-                f"Error: {str(e)}\n\n"
-                f"Code context:\n{error_context['code_context']}"
-            )
-            self.logger.error(
-                message="{error}",
-                tag="ERROR",
-                params={"error": str(error_message)},
-                boxes=["error"],
-            )
-            raise
-        finally:
-            async with self.pool_lock:
-                if task_id in self.connection_pool:
-                    await self.connection_pool[task_id].close()
-                    del self.connection_pool[task_id]
-            self.connection_semaphore.release()
-
-    async def execute_with_retry(self, operation, *args):
-        """Execute database operations with retry logic"""
-        for attempt in range(self.max_retries):
-            try:
-                async with self.get_connection() as db:
-                    result = await operation(db, *args)
-                    await db.commit()
-                    return result
-            except Exception as e:
-                if attempt == self.max_retries - 1:
-                    self.logger.error(
-                        message="Operation failed after {retries} attempts: {error}",
-                        tag="ERROR",
-                        force_verbose=True,
-                        params={"retries": self.max_retries, "error": str(e)},
-                    )
-                    raise
-                await asyncio.sleep(1 * (attempt + 1))  # Exponential backoff
 
     async def ainit_db(self):
-        """Initialize database schema"""
-        async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
-            await db.execute(
-                """
+        async with aiosqlite.connect(self.db_path) as db:
+            await db.execute('''
                 CREATE TABLE IF NOT EXISTS crawled_data (
                     url TEXT PRIMARY KEY,
                     html TEXT,
@@ -239,326 +25,90 @@ class AsyncDatabaseManager:
                     media TEXT DEFAULT "{}",
                     links TEXT DEFAULT "{}",
                     metadata TEXT DEFAULT "{}",
-                    screenshot TEXT DEFAULT "",
-                    response_headers TEXT DEFAULT "{}",
-                    downloaded_files TEXT DEFAULT "{}"  -- New column added
+                    screenshot TEXT DEFAULT ""
                 )
-            """
-            )
+            ''')
             await db.commit()
+        await self.update_db_schema()
 
     async def update_db_schema(self):
-        """Update database schema if needed"""
-        async with aiosqlite.connect(self.db_path, timeout=30.0) as db:
+        async with aiosqlite.connect(self.db_path) as db:
+            # Check if the 'media' column exists
             cursor = await db.execute("PRAGMA table_info(crawled_data)")
             columns = await cursor.fetchall()
             column_names = [column[1] for column in columns]
-
-            # List of new columns to add
-            new_columns = [
-                "media",
-                "links",
-                "metadata",
-                "screenshot",
-                "response_headers",
-                "downloaded_files",
-            ]
-
-            for column in new_columns:
+            
+            if 'media' not in column_names:
+                await self.aalter_db_add_column('media')
+            
+            # Check for other missing columns and add them if necessary
+            for column in ['links', 'metadata', 'screenshot']:
                 if column not in column_names:
-                    await self.aalter_db_add_column(column, db)
-            await db.commit()
-
-    async def aalter_db_add_column(self, new_column: str, db):
-        """Add new column to the database"""
-        if new_column == "response_headers":
-            await db.execute(
-                f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT "{{}}"'
-            )
-        else:
-            await db.execute(
-                f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""'
-            )
-        self.logger.info(
-            message="Added column '{column}' to the database",
-            tag="INIT",
-            params={"column": new_column},
-        )
-
-    async def aget_cached_url(self, url: str) -> Optional[CrawlResult]:
-        """Retrieve cached URL data as CrawlResult"""
-
-        async def _get(db):
-            async with db.execute(
-                "SELECT * FROM crawled_data WHERE url = ?", (url,)
-            ) as cursor:
-                row = await cursor.fetchone()
-                if not row:
-                    return None
-
-                # Get column names
-                columns = [description[0] for description in cursor.description]
-                # Create dict from row data
-                row_dict = dict(zip(columns, row))
-
-                # Load content from files using stored hashes
-                content_fields = {
-                    "html": row_dict["html"],
-                    "cleaned_html": row_dict["cleaned_html"],
-                    "markdown": row_dict["markdown"],
-                    "extracted_content": row_dict["extracted_content"],
-                    "screenshot": row_dict["screenshot"],
-                    "screenshots": row_dict["screenshot"],
-                }
-
-                for field, hash_value in content_fields.items():
-                    if hash_value:
-                        content = await self._load_content(
-                            hash_value,
-                            field.split("_")[0],  # Get content type from field name
-                        )
-                        row_dict[field] = content or ""
-                    else:
-                        row_dict[field] = ""
-
-                # Parse JSON fields
-                json_fields = [
-                    "media",
-                    "links",
-                    "metadata",
-                    "response_headers",
-                    "markdown",
-                ]
-                for field in json_fields:
-                    try:
-                        row_dict[field] = (
-                            json.loads(row_dict[field]) if row_dict[field] else {}
-                        )
-                    except json.JSONDecodeError:
-                        # Very UGLY, never mention it to me please
-                        if field == "markdown" and isinstance(row_dict[field], str):
-                            row_dict[field] = MarkdownGenerationResult(
-                                raw_markdown=row_dict[field] or "",
-                                markdown_with_citations="",
-                                references_markdown="",
-                                fit_markdown="",
-                                fit_html="",
-                            )
-                        else:
-                            row_dict[field] = {}
-
-                if isinstance(row_dict["markdown"], Dict):
-                    if row_dict["markdown"].get("raw_markdown"):
-                        row_dict["markdown"] = row_dict["markdown"]["raw_markdown"]
-
-                # Parse downloaded_files
-                try:
-                    row_dict["downloaded_files"] = (
-                        json.loads(row_dict["downloaded_files"])
-                        if row_dict["downloaded_files"]
-                        else []
-                    )
-                except json.JSONDecodeError:
-                    row_dict["downloaded_files"] = []
-
-                # Remove any fields not in CrawlResult model
-                valid_fields = CrawlResult.__annotations__.keys()
-                filtered_dict = {k: v for k, v in row_dict.items() if k in valid_fields}
-                filtered_dict["markdown"] = row_dict["markdown"]
-                return CrawlResult(**filtered_dict)
+                    await self.aalter_db_add_column(column)
 
+    async def aalter_db_add_column(self, new_column: str):
         try:
-            return await self.execute_with_retry(_get)
+            async with aiosqlite.connect(self.db_path) as db:
+                await db.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""')
+                await db.commit()
+            print(f"Added column '{new_column}' to the database.")
         except Exception as e:
-            self.logger.error(
-                message="Error retrieving cached URL: {error}",
-                tag="ERROR",
-                force_verbose=True,
-                params={"error": str(e)},
-            )
+            print(f"Error altering database to add {new_column} column: {e}")
+
+    async def aget_cached_url(self, url: str) -> Optional[Tuple[str, str, str, str, str, str, str, bool, str]]:
+        try:
+            async with aiosqlite.connect(self.db_path) as db:
+                async with db.execute('SELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?', (url,)) as cursor:
+                    return await cursor.fetchone()
+        except Exception as e:
+            print(f"Error retrieving cached URL: {e}")
             return None
 
-    async def acache_url(self, result: CrawlResult):
-        """Cache CrawlResult data"""
-        # Store content files and get hashes
-        content_map = {
-            "html": (result.html, "html"),
-            "cleaned_html": (result.cleaned_html or "", "cleaned"),
-            "markdown": None,
-            "extracted_content": (result.extracted_content or "", "extracted"),
-            "screenshot": (result.screenshot or "", "screenshots"),
-        }
-
+    async def acache_url(self, url: str, html: str, cleaned_html: str, markdown: str, extracted_content: str, success: bool, media: str = "{}", links: str = "{}", metadata: str = "{}", screenshot: str = ""):
         try:
-            if isinstance(result.markdown, StringCompatibleMarkdown):
-                content_map["markdown"] = (
-                    result.markdown,
-                    "markdown",
-                )
-            elif isinstance(result.markdown, MarkdownGenerationResult):
-                content_map["markdown"] = (
-                    result.markdown.model_dump_json(),
-                    "markdown",
-                )
-            elif isinstance(result.markdown, str):
-                markdown_result = MarkdownGenerationResult(raw_markdown=result.markdown)
-                content_map["markdown"] = (
-                    markdown_result.model_dump_json(),
-                    "markdown",
-                )
-            else:
-                content_map["markdown"] = (
-                    MarkdownGenerationResult().model_dump_json(),
-                    "markdown",
-                )
+            async with aiosqlite.connect(self.db_path) as db:
+                await db.execute('''
+                    INSERT INTO crawled_data (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    ON CONFLICT(url) DO UPDATE SET
+                        html = excluded.html,
+                        cleaned_html = excluded.cleaned_html,
+                        markdown = excluded.markdown,
+                        extracted_content = excluded.extracted_content,
+                        success = excluded.success,
+                        media = excluded.media,      
+                        links = excluded.links,    
+                        metadata = excluded.metadata,      
+                        screenshot = excluded.screenshot
+                ''', (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot))
+                await db.commit()
         except Exception as e:
-            self.logger.warning(
-                message=f"Error processing markdown content: {str(e)}", tag="WARNING"
-            )
-            # Fallback to empty markdown result
-            content_map["markdown"] = (
-                MarkdownGenerationResult().model_dump_json(),
-                "markdown",
-            )
-
-        content_hashes = {}
-        for field, (content, content_type) in content_map.items():
-            content_hashes[field] = await self._store_content(content, content_type)
-
-        async def _cache(db):
-            await db.execute(
-                """
-                INSERT INTO crawled_data (
-                    url, html, cleaned_html, markdown,
-                    extracted_content, success, media, links, metadata,
-                    screenshot, response_headers, downloaded_files
-                )
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-                ON CONFLICT(url) DO UPDATE SET
-                    html = excluded.html,
-                    cleaned_html = excluded.cleaned_html,
-                    markdown = excluded.markdown,
-                    extracted_content = excluded.extracted_content,
-                    success = excluded.success,
-                    media = excluded.media,
-                    links = excluded.links,
-                    metadata = excluded.metadata,
-                    screenshot = excluded.screenshot,
-                    response_headers = excluded.response_headers,
-                    downloaded_files = excluded.downloaded_files
-            """,
-                (
-                    result.url,
-                    content_hashes["html"],
-                    content_hashes["cleaned_html"],
-                    content_hashes["markdown"],
-                    content_hashes["extracted_content"],
-                    result.success,
-                    json.dumps(result.media),
-                    json.dumps(result.links),
-                    json.dumps(result.metadata or {}),
-                    content_hashes["screenshot"],
-                    json.dumps(result.response_headers or {}),
-                    json.dumps(result.downloaded_files or []),
-                ),
-            )
-
-        try:
-            await self.execute_with_retry(_cache)
-        except Exception as e:
-            self.logger.error(
-                message="Error caching URL: {error}",
-                tag="ERROR",
-                force_verbose=True,
-                params={"error": str(e)},
-            )
+            print(f"Error caching URL: {e}")
 
     async def aget_total_count(self) -> int:
-        """Get total number of cached URLs"""
-
-        async def _count(db):
-            async with db.execute("SELECT COUNT(*) FROM crawled_data") as cursor:
-                result = await cursor.fetchone()
-                return result[0] if result else 0
-
         try:
-            return await self.execute_with_retry(_count)
+            async with aiosqlite.connect(self.db_path) as db:
+                async with db.execute('SELECT COUNT(*) FROM crawled_data') as cursor:
+                    result = await cursor.fetchone()
+                    return result[0] if result else 0
         except Exception as e:
-            self.logger.error(
-                message="Error getting total count: {error}",
-                tag="ERROR",
-                force_verbose=True,
-                params={"error": str(e)},
-            )
+            print(f"Error getting total count: {e}")
             return 0
 
     async def aclear_db(self):
-        """Clear all data from the database"""
-
-        async def _clear(db):
-            await db.execute("DELETE FROM crawled_data")
-
         try:
-            await self.execute_with_retry(_clear)
+            async with aiosqlite.connect(self.db_path) as db:
+                await db.execute('DELETE FROM crawled_data')
+                await db.commit()
         except Exception as e:
-            self.logger.error(
-                message="Error clearing database: {error}",
-                tag="ERROR",
-                force_verbose=True,
-                params={"error": str(e)},
-            )
+            print(f"Error clearing database: {e}")
 
     async def aflush_db(self):
-        """Drop the entire table"""
-
-        async def _flush(db):
-            await db.execute("DROP TABLE IF EXISTS crawled_data")
-
         try:
-            await self.execute_with_retry(_flush)
+            async with aiosqlite.connect(self.db_path) as db:
+                await db.execute('DROP TABLE IF EXISTS crawled_data')
+                await db.commit()
         except Exception as e:
-            self.logger.error(
-                message="Error flushing database: {error}",
-                tag="ERROR",
-                force_verbose=True,
-                params={"error": str(e)},
-            )
+            print(f"Error flushing database: {e}")
 
-    async def _store_content(self, content: str, content_type: str) -> str:
-        """Store content in filesystem and return hash"""
-        if not content:
-            return ""
-
-        content_hash = generate_content_hash(content)
-        file_path = os.path.join(self.content_paths[content_type], content_hash)
-
-        # Only write if file doesn't exist
-        if not os.path.exists(file_path):
-            async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
-                await f.write(content)
-
-        return content_hash
-
-    async def _load_content(
-        self, content_hash: str, content_type: str
-    ) -> Optional[str]:
-        """Load content from filesystem by hash"""
-        if not content_hash:
-            return None
-
-        file_path = os.path.join(self.content_paths[content_type], content_hash)
-        try:
-            async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
-                return await f.read()
-        except:
-            self.logger.error(
-                message="Failed to load content: {file_path}",
-                tag="ERROR",
-                force_verbose=True,
-                params={"file_path": file_path},
-            )
-            return None
-
-
-# Create a singleton instance
-async_db_manager = AsyncDatabaseManager()
+async_db_manager = AsyncDatabaseManager()

crawl4ai/async_dispatcher.py

@@ -1,771 +0,0 @@
-from typing import Dict, Optional, List, Tuple, Union
-from .async_configs import CrawlerRunConfig
-from .models import (
-    CrawlResult,
-    CrawlerTaskResult,
-    CrawlStatus,
-    DomainState,
-)
-
-from .components.crawler_monitor import CrawlerMonitor
-
-from .types import AsyncWebCrawler
-
-from collections.abc import AsyncGenerator
-
-import time
-import psutil
-import asyncio
-import uuid
-
-from urllib.parse import urlparse
-import random
-from abc import ABC, abstractmethod
-
-from .utils import get_true_memory_usage_percent
-
-
-class RateLimiter:
-    def __init__(
-        self,
-        base_delay: Tuple[float, float] = (1.0, 3.0),
-        max_delay: float = 60.0,
-        max_retries: int = 3,
-        rate_limit_codes: List[int] = None,
-    ):
-        self.base_delay = base_delay
-        self.max_delay = max_delay
-        self.max_retries = max_retries
-        self.rate_limit_codes = rate_limit_codes or [429, 503]
-        self.domains: Dict[str, DomainState] = {}
-
-    def get_domain(self, url: str) -> str:
-        return urlparse(url).netloc
-
-    async def wait_if_needed(self, url: str) -> None:
-        domain = self.get_domain(url)
-        state = self.domains.get(domain)
-
-        if not state:
-            self.domains[domain] = DomainState()
-            state = self.domains[domain]
-
-        now = time.time()
-        if state.last_request_time:
-            wait_time = max(0, state.current_delay - (now - state.last_request_time))
-            if wait_time > 0:
-                await asyncio.sleep(wait_time)
-
-        # Random delay within base range if no current delay
-        if state.current_delay == 0:
-            state.current_delay = random.uniform(*self.base_delay)
-
-        state.last_request_time = time.time()
-
-    def update_delay(self, url: str, status_code: int) -> bool:
-        domain = self.get_domain(url)
-        state = self.domains[domain]
-
-        if status_code in self.rate_limit_codes:
-            state.fail_count += 1
-            if state.fail_count > self.max_retries:
-                return False
-
-            # Exponential backoff with random jitter
-            state.current_delay = min(
-                state.current_delay * 2 * random.uniform(0.75, 1.25), self.max_delay
-            )
-        else:
-            # Gradually reduce delay on success
-            state.current_delay = max(
-                random.uniform(*self.base_delay), state.current_delay * 0.75
-            )
-            state.fail_count = 0
-
-        return True
-
-
-
-class BaseDispatcher(ABC):
-    def __init__(
-        self,
-        rate_limiter: Optional[RateLimiter] = None,
-        monitor: Optional[CrawlerMonitor] = None,
-    ):
-        self.crawler = None
-        self._domain_last_hit: Dict[str, float] = {}
-        self.concurrent_sessions = 0
-        self.rate_limiter = rate_limiter
-        self.monitor = monitor
-
-    def select_config(self, url: str, configs: Union[CrawlerRunConfig, List[CrawlerRunConfig]]) -> Optional[CrawlerRunConfig]:
-        """Select the appropriate config for a given URL.
-        
-        Args:
-            url: The URL to match against
-            configs: Single config or list of configs to choose from
-            
-        Returns:
-            The matching config, or None if no match found
-        """
-        # Single config - return as is
-        if isinstance(configs, CrawlerRunConfig):
-            return configs
-        
-        # Empty list - return None
-        if not configs:
-            return None
-        
-        # Find first matching config
-        for config in configs:
-            if config.is_match(url):
-                return config
-        
-        # No match found - return None to indicate URL should be skipped
-        return None
-
-    @abstractmethod
-    async def crawl_url(
-        self,
-        url: str,
-        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
-        task_id: str,
-        monitor: Optional[CrawlerMonitor] = None,
-    ) -> CrawlerTaskResult:
-        pass
-
-    @abstractmethod
-    async def run_urls(
-        self,
-        urls: List[str],
-        crawler: AsyncWebCrawler,  # noqa: F821
-        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
-        monitor: Optional[CrawlerMonitor] = None,
-    ) -> List[CrawlerTaskResult]:
-        pass
-
-
-class MemoryAdaptiveDispatcher(BaseDispatcher):
-    def __init__(
-        self,
-        memory_threshold_percent: float = 90.0,
-        critical_threshold_percent: float = 95.0,  # New critical threshold
-        recovery_threshold_percent: float = 85.0,  # New recovery threshold
-        check_interval: float = 1.0,
-        max_session_permit: int = 20,
-        fairness_timeout: float = 600.0,  # 10 minutes before prioritizing long-waiting URLs
-        memory_wait_timeout: Optional[float] = 600.0,
-        rate_limiter: Optional[RateLimiter] = None,
-        monitor: Optional[CrawlerMonitor] = None,
-    ):
-        super().__init__(rate_limiter, monitor)
-        self.memory_threshold_percent = memory_threshold_percent
-        self.critical_threshold_percent = critical_threshold_percent
-        self.recovery_threshold_percent = recovery_threshold_percent
-        self.check_interval = check_interval
-        self.max_session_permit = max_session_permit
-        self.fairness_timeout = fairness_timeout
-        self.memory_wait_timeout = memory_wait_timeout
-        self.result_queue = asyncio.Queue()
-        self.task_queue = asyncio.PriorityQueue()  # Priority queue for better management
-        self.memory_pressure_mode = False  # Flag to indicate when we're in memory pressure mode
-        self.current_memory_percent = 0.0  # Track current memory usage
-        self._high_memory_start_time: Optional[float] = None
-        
-    async def _memory_monitor_task(self):
-        """Background task to continuously monitor memory usage and update state"""
-        while True:
-            self.current_memory_percent = get_true_memory_usage_percent()
-
-            # Enter memory pressure mode if we cross the threshold
-            if self.current_memory_percent >= self.memory_threshold_percent:
-                if not self.memory_pressure_mode:
-                    self.memory_pressure_mode = True
-                    self._high_memory_start_time = time.time()
-                    if self.monitor:
-                        self.monitor.update_memory_status("PRESSURE")
-                else:
-                    if self._high_memory_start_time is None:
-                        self._high_memory_start_time = time.time()
-                    if (
-                        self.memory_wait_timeout is not None
-                        and self._high_memory_start_time is not None
-                        and time.time() - self._high_memory_start_time >= self.memory_wait_timeout
-                    ):
-                        raise MemoryError(
-                            "Memory usage exceeded threshold for"
-                            f" {self.memory_wait_timeout} seconds"
-                        )
-
-            # Exit memory pressure mode if we go below recovery threshold
-            elif self.memory_pressure_mode and self.current_memory_percent <= self.recovery_threshold_percent:
-                self.memory_pressure_mode = False
-                self._high_memory_start_time = None
-                if self.monitor:
-                    self.monitor.update_memory_status("NORMAL")
-            elif self.current_memory_percent < self.memory_threshold_percent:
-                self._high_memory_start_time = None
-            
-            # In critical mode, we might need to take more drastic action
-            if self.current_memory_percent >= self.critical_threshold_percent:
-                if self.monitor:
-                    self.monitor.update_memory_status("CRITICAL")
-                # We could implement additional memory-saving measures here
-                
-            await asyncio.sleep(self.check_interval)
-    
-    def _get_priority_score(self, wait_time: float, retry_count: int) -> float:
-        """Calculate priority score (lower is higher priority)
-        - URLs waiting longer than fairness_timeout get higher priority
-        - More retry attempts decreases priority
-        """
-        if wait_time > self.fairness_timeout:
-            # High priority for long-waiting URLs
-            return -wait_time
-        # Standard priority based on retries
-        return retry_count
-    
-    async def crawl_url(
-        self,
-        url: str,
-        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
-        task_id: str,
-        retry_count: int = 0,
-    ) -> CrawlerTaskResult:
-        start_time = time.time()
-        error_message = ""
-        memory_usage = peak_memory = 0.0
-        
-        # Select appropriate config for this URL
-        selected_config = self.select_config(url, config)
-        
-        # If no config matches, return failed result
-        if selected_config is None:
-            error_message = f"No matching configuration found for URL: {url}"
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id, 
-                    status=CrawlStatus.FAILED,
-                    error_message=error_message
-                )
-            
-            return CrawlerTaskResult(
-                task_id=task_id,
-                url=url,
-                result=CrawlResult(
-                    url=url, 
-                    html="", 
-                    metadata={"status": "no_config_match"}, 
-                    success=False, 
-                    error_message=error_message
-                ),
-                memory_usage=0,
-                peak_memory=0,
-                start_time=start_time,
-                end_time=time.time(),
-                error_message=error_message,
-                retry_count=retry_count
-            )
-        
-        # Get starting memory for accurate measurement
-        process = psutil.Process()
-        start_memory = process.memory_info().rss / (1024 * 1024)
-        
-        try:
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id, 
-                    status=CrawlStatus.IN_PROGRESS, 
-                    start_time=start_time,
-                    retry_count=retry_count
-                )
-                
-            self.concurrent_sessions += 1
-            
-            if self.rate_limiter:
-                await self.rate_limiter.wait_if_needed(url)
-                
-            # Check if we're in critical memory state
-            if self.current_memory_percent >= self.critical_threshold_percent:
-                # Requeue this task with increased priority and retry count
-                enqueue_time = time.time()
-                priority = self._get_priority_score(enqueue_time - start_time, retry_count + 1)
-                await self.task_queue.put((priority, (url, task_id, retry_count + 1, enqueue_time)))
-                
-                # Update monitoring
-                if self.monitor:
-                    self.monitor.update_task(
-                        task_id,
-                        status=CrawlStatus.QUEUED,
-                        error_message="Requeued due to critical memory pressure"
-                    )
-                
-                # Return placeholder result with requeued status
-                return CrawlerTaskResult(
-                    task_id=task_id,
-                    url=url,
-                    result=CrawlResult(
-                        url=url, html="", metadata={"status": "requeued"}, 
-                        success=False, error_message="Requeued due to critical memory pressure"
-                    ),
-                    memory_usage=0,
-                    peak_memory=0,
-                    start_time=start_time,
-                    end_time=time.time(),
-                    error_message="Requeued due to critical memory pressure",
-                    retry_count=retry_count + 1
-                )
-            
-            # Execute the crawl with selected config
-            result = await self.crawler.arun(url, config=selected_config, session_id=task_id)
-            
-            # Measure memory usage
-            end_memory = process.memory_info().rss / (1024 * 1024)
-            memory_usage = peak_memory = end_memory - start_memory
-            
-            # Handle rate limiting
-            if self.rate_limiter and result.status_code:
-                if not self.rate_limiter.update_delay(url, result.status_code):
-                    error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
-                    if self.monitor:
-                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                        
-            # Update status based on result
-            if not result.success:
-                error_message = result.error_message
-                if self.monitor:
-                    self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-            elif self.monitor:
-                self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
-                
-        except Exception as e:
-            error_message = str(e)
-            if self.monitor:
-                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-            result = CrawlResult(
-                url=url, html="", metadata={}, success=False, error_message=str(e)
-            )
-            
-        finally:
-            end_time = time.time()
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id,
-                    end_time=end_time,
-                    memory_usage=memory_usage,
-                    peak_memory=peak_memory,
-                    error_message=error_message,
-                    retry_count=retry_count
-                )
-            self.concurrent_sessions -= 1
-            
-        return CrawlerTaskResult(
-            task_id=task_id,
-            url=url,
-            result=result,
-            memory_usage=memory_usage,
-            peak_memory=peak_memory,
-            start_time=start_time,
-            end_time=end_time,
-            error_message=error_message,
-            retry_count=retry_count
-        )
-        
-    async def run_urls(
-        self,
-        urls: List[str],
-        crawler: AsyncWebCrawler,
-        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
-    ) -> List[CrawlerTaskResult]:
-        self.crawler = crawler
-        
-        # Start the memory monitor task
-        memory_monitor = asyncio.create_task(self._memory_monitor_task())
-        
-        if self.monitor:
-            self.monitor.start()
-            
-        results = []
-
-        try:
-            # Initialize task queue
-            for url in urls:
-                task_id = str(uuid.uuid4())
-                if self.monitor:
-                    self.monitor.add_task(task_id, url)
-                # Add to queue with initial priority 0, retry count 0, and current time
-                await self.task_queue.put((0, (url, task_id, 0, time.time())))
-
-            active_tasks = []
-
-            # Process until both queues are empty
-            while not self.task_queue.empty() or active_tasks:
-                if memory_monitor.done():
-                    exc = memory_monitor.exception()
-                    if exc:
-                        for t in active_tasks:
-                            t.cancel()
-                        raise exc
-
-                # If memory pressure is low, greedily fill all available slots
-                if not self.memory_pressure_mode:
-                    slots = self.max_session_permit - len(active_tasks)
-                    while slots > 0:
-                        try:
-                            # Use get_nowait() to immediately get tasks without blocking
-                            priority, (url, task_id, retry_count, enqueue_time) = self.task_queue.get_nowait()
-                            
-                            # Create and start the task
-                            task = asyncio.create_task(
-                                self.crawl_url(url, config, task_id, retry_count)
-                            )
-                            active_tasks.append(task)
-                            
-                            # Update waiting time in monitor
-                            if self.monitor:
-                                wait_time = time.time() - enqueue_time
-                                self.monitor.update_task(
-                                    task_id, 
-                                    wait_time=wait_time,
-                                    status=CrawlStatus.IN_PROGRESS
-                                )
-                            
-                            slots -= 1
-                            
-                        except asyncio.QueueEmpty:
-                            # No more tasks in queue, exit the loop
-                            break
-                        
-                # Wait for completion even if queue is starved
-                if active_tasks:
-                    done, pending = await asyncio.wait(
-                        active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
-                    )
-                    
-                    # Process completed tasks
-                    for completed_task in done:
-                        result = await completed_task
-                        results.append(result)
-                        
-                    # Update active tasks list
-                    active_tasks = list(pending)
-                else:
-                    # If no active tasks but still waiting, sleep briefly
-                    await asyncio.sleep(self.check_interval / 2)
-                    
-                # Update priorities for waiting tasks if needed
-                await self._update_queue_priorities()
-
-        except Exception as e:
-            if self.monitor:
-                self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")                
-        
-        finally:
-            # Clean up
-            memory_monitor.cancel()
-            if self.monitor:
-                self.monitor.stop()
-            return results
-                
-    async def _update_queue_priorities(self):
-        """Periodically update priorities of items in the queue to prevent starvation"""
-        # Skip if queue is empty
-        if self.task_queue.empty():
-            return
-            
-        # Use a drain-and-refill approach to update all priorities
-        temp_items = []
-        
-        # Drain the queue (with a safety timeout to prevent blocking)
-        try:
-            drain_start = time.time()
-            while not self.task_queue.empty() and time.time() - drain_start < 5.0:  # 5 second safety timeout
-                try:
-                    # Get item from queue with timeout
-                    priority, (url, task_id, retry_count, enqueue_time) = await asyncio.wait_for(
-                        self.task_queue.get(), timeout=0.1
-                    )
-                    
-                    # Calculate new priority based on current wait time
-                    current_time = time.time()
-                    wait_time = current_time - enqueue_time
-                    new_priority = self._get_priority_score(wait_time, retry_count)
-                    
-                    # Store with updated priority
-                    temp_items.append((new_priority, (url, task_id, retry_count, enqueue_time)))
-                    
-                    # Update monitoring stats for this task
-                    if self.monitor and task_id in self.monitor.stats:
-                        self.monitor.update_task(task_id, wait_time=wait_time)
-                        
-                except asyncio.TimeoutError:
-                    # Queue might be empty or very slow
-                    break
-        except Exception as e:
-            # If anything goes wrong, make sure we refill the queue with what we've got
-            self.monitor.update_memory_status(f"QUEUE_ERROR: {str(e)}")
-        
-        # Calculate queue statistics
-        if temp_items and self.monitor:
-            total_queued = len(temp_items)
-            wait_times = [item[1][3] for item in temp_items]
-            highest_wait_time = time.time() - min(wait_times) if wait_times else 0
-            avg_wait_time = sum(time.time() - t for t in wait_times) / len(wait_times) if wait_times else 0
-            
-            # Update queue statistics in monitor
-            self.monitor.update_queue_statistics(
-                total_queued=total_queued,
-                highest_wait_time=highest_wait_time,
-                avg_wait_time=avg_wait_time
-            )
-        
-        # Sort by priority (lowest number = highest priority)
-        temp_items.sort(key=lambda x: x[0])
-        
-        # Refill the queue with updated priorities
-        for item in temp_items:
-            await self.task_queue.put(item)
-                
-    async def run_urls_stream(
-        self,
-        urls: List[str],
-        crawler: AsyncWebCrawler,
-        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
-    ) -> AsyncGenerator[CrawlerTaskResult, None]:
-        self.crawler = crawler
-        
-        # Start the memory monitor task
-        memory_monitor = asyncio.create_task(self._memory_monitor_task())
-        
-        if self.monitor:
-            self.monitor.start()
-            
-        try:
-            # Initialize task queue
-            for url in urls:
-                task_id = str(uuid.uuid4())
-                if self.monitor:
-                    self.monitor.add_task(task_id, url)
-                # Add to queue with initial priority 0, retry count 0, and current time
-                await self.task_queue.put((0, (url, task_id, 0, time.time())))
-                
-            active_tasks = []
-            completed_count = 0
-            total_urls = len(urls)
-
-            while completed_count < total_urls:
-                if memory_monitor.done():
-                    exc = memory_monitor.exception()
-                    if exc:
-                        for t in active_tasks:
-                            t.cancel()
-                        raise exc
-                # If memory pressure is low, greedily fill all available slots
-                if not self.memory_pressure_mode:
-                    slots = self.max_session_permit - len(active_tasks)
-                    while slots > 0:
-                        try:
-                            # Use get_nowait() to immediately get tasks without blocking
-                            priority, (url, task_id, retry_count, enqueue_time) = self.task_queue.get_nowait()
-                            
-                            # Create and start the task
-                            task = asyncio.create_task(
-                                self.crawl_url(url, config, task_id, retry_count)
-                            )
-                            active_tasks.append(task)
-                            
-                            # Update waiting time in monitor
-                            if self.monitor:
-                                wait_time = time.time() - enqueue_time
-                                self.monitor.update_task(
-                                    task_id, 
-                                    wait_time=wait_time,
-                                    status=CrawlStatus.IN_PROGRESS
-                                )
-                            
-                            slots -= 1
-                            
-                        except asyncio.QueueEmpty:
-                            # No more tasks in queue, exit the loop
-                            break
-                        
-                # Process completed tasks and yield results
-                if active_tasks:
-                    done, pending = await asyncio.wait(
-                        active_tasks, timeout=0.1, return_when=asyncio.FIRST_COMPLETED
-                    )
-                    
-                    for completed_task in done:
-                        result = await completed_task
-                        
-                        # Only count as completed if it wasn't requeued
-                        if "requeued" not in result.error_message:
-                            completed_count += 1
-                            yield result
-                        
-                    # Update active tasks list
-                    active_tasks = list(pending)
-                else:
-                    # If no active tasks but still waiting, sleep briefly
-                    await asyncio.sleep(self.check_interval / 2)
-                
-                # Update priorities for waiting tasks if needed
-                await self._update_queue_priorities()
-                
-        finally:
-            # Clean up
-            memory_monitor.cancel()
-            if self.monitor:
-                self.monitor.stop()
-                
-
-class SemaphoreDispatcher(BaseDispatcher):
-    def __init__(
-        self,
-        semaphore_count: int = 5,
-        max_session_permit: int = 20,
-        rate_limiter: Optional[RateLimiter] = None,
-        monitor: Optional[CrawlerMonitor] = None,
-    ):
-        super().__init__(rate_limiter, monitor)
-        self.semaphore_count = semaphore_count
-        self.max_session_permit = max_session_permit
-
-    async def crawl_url(
-        self,
-        url: str,
-        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
-        task_id: str,
-        semaphore: asyncio.Semaphore = None,
-    ) -> CrawlerTaskResult:
-        start_time = time.time()
-        error_message = ""
-        memory_usage = peak_memory = 0.0
-
-        # Select appropriate config for this URL
-        selected_config = self.select_config(url, config)
-        
-        # If no config matches, return failed result
-        if selected_config is None:
-            error_message = f"No matching configuration found for URL: {url}"
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id, 
-                    status=CrawlStatus.FAILED,
-                    error_message=error_message
-                )
-            
-            return CrawlerTaskResult(
-                task_id=task_id,
-                url=url,
-                result=CrawlResult(
-                    url=url, 
-                    html="", 
-                    metadata={"status": "no_config_match"}, 
-                    success=False, 
-                    error_message=error_message
-                ),
-                memory_usage=0,
-                peak_memory=0,
-                start_time=start_time,
-                end_time=time.time(),
-                error_message=error_message
-            )
-
-        try:
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id, status=CrawlStatus.IN_PROGRESS, start_time=start_time
-                )
-
-            if self.rate_limiter:
-                await self.rate_limiter.wait_if_needed(url)
-
-            async with semaphore:
-                process = psutil.Process()
-                start_memory = process.memory_info().rss / (1024 * 1024)
-                result = await self.crawler.arun(url, config=selected_config, session_id=task_id)
-                end_memory = process.memory_info().rss / (1024 * 1024)
-
-                memory_usage = peak_memory = end_memory - start_memory
-
-                if self.rate_limiter and result.status_code:
-                    if not self.rate_limiter.update_delay(url, result.status_code):
-                        error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
-                        if self.monitor:
-                            self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                        return CrawlerTaskResult(
-                            task_id=task_id,
-                            url=url,
-                            result=result,
-                            memory_usage=memory_usage,
-                            peak_memory=peak_memory,
-                            start_time=start_time,
-                            end_time=time.time(),
-                            error_message=error_message,
-                        )
-
-                if not result.success:
-                    error_message = result.error_message
-                    if self.monitor:
-                        self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                elif self.monitor:
-                    self.monitor.update_task(task_id, status=CrawlStatus.COMPLETED)
-
-        except Exception as e:
-            error_message = str(e)
-            if self.monitor:
-                self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-            result = CrawlResult(
-                url=url, html="", metadata={}, success=False, error_message=str(e)
-            )
-
-        finally:
-            end_time = time.time()
-            if self.monitor:
-                self.monitor.update_task(
-                    task_id,
-                    end_time=end_time,
-                    memory_usage=memory_usage,
-                    peak_memory=peak_memory,
-                    error_message=error_message,
-                )
-
-        return CrawlerTaskResult(
-            task_id=task_id,
-            url=url,
-            result=result,
-            memory_usage=memory_usage,
-            peak_memory=peak_memory,
-            start_time=start_time,
-            end_time=end_time,
-            error_message=error_message,
-        )
-
-    async def run_urls(
-        self,
-        crawler: AsyncWebCrawler,  # noqa: F821
-        urls: List[str],
-        config: Union[CrawlerRunConfig, List[CrawlerRunConfig]],
-    ) -> List[CrawlerTaskResult]:
-        self.crawler = crawler
-        if self.monitor:
-            self.monitor.start()
-
-        try:
-            semaphore = asyncio.Semaphore(self.semaphore_count)
-            tasks = []
-
-            for url in urls:
-                task_id = str(uuid.uuid4())
-                if self.monitor:
-                    self.monitor.add_task(task_id, url)
-                task = asyncio.create_task(
-                    self.crawl_url(url, config, task_id, semaphore)
-                )
-                tasks.append(task)
-
-            return await asyncio.gather(*tasks, return_exceptions=True)
-        finally:
-            if self.monitor:
-                self.monitor.stop()

crawl4ai/async_logger.py

@@ -1,374 +0,0 @@
-from abc import ABC, abstractmethod
-from enum import Enum
-from typing import Optional, Dict, Any, List
-import os
-from datetime import datetime
-from urllib.parse import unquote
-from rich.console import Console
-from rich.text import Text
-from .utils import create_box_message
-
-
-class LogLevel(Enum):
-    DEFAULT = 0
-    DEBUG = 1
-    INFO = 2
-    SUCCESS = 3
-    WARNING = 4
-    ERROR = 5
-    CRITICAL = 6
-    ALERT = 7
-    NOTICE = 8
-    EXCEPTION = 9
-    FATAL = 10
-    
-
-    def __str__(self):
-        return self.name.lower()
-
-class LogColor(str, Enum):
-    """Enum for log colors."""
-
-    DEBUG = "bright_black"
-    INFO = "cyan"
-    SUCCESS = "green"
-    WARNING = "yellow"
-    ERROR = "red"
-    CYAN = "cyan"
-    GREEN = "green"
-    YELLOW = "yellow"
-    MAGENTA = "magenta"
-    DIM_MAGENTA = "dim magenta"
-    RED = "red"
-
-    def __str__(self):
-        """Automatically convert rich color to string."""
-        return self.value
-
-
-class AsyncLoggerBase(ABC):
-    @abstractmethod
-    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
-        pass
-
-    @abstractmethod
-    def info(self, message: str, tag: str = "INFO", **kwargs):
-        pass
-
-    @abstractmethod
-    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
-        pass
-
-    @abstractmethod
-    def warning(self, message: str, tag: str = "WARNING", **kwargs):
-        pass
-
-    @abstractmethod
-    def error(self, message: str, tag: str = "ERROR", **kwargs):
-        pass
-
-    @abstractmethod
-    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 100):
-        pass
-
-    @abstractmethod
-    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 100):
-        pass
-
-
-class AsyncLogger(AsyncLoggerBase):
-    """
-    Asynchronous logger with support for colored console output and file logging.
-    Supports templated messages with colored components.
-    """
-
-    DEFAULT_ICONS = {
-        "INIT": "→",
-        "READY": "✓",
-        "FETCH": "↓",
-        "SCRAPE": "◆",
-        "EXTRACT": "■",
-        "COMPLETE": "●",
-        "ERROR": "×",
-        "DEBUG": "⋯",
-        "INFO": "ℹ",
-        "WARNING": "⚠",
-        "SUCCESS": "✔",
-        "CRITICAL": "‼",
-        "ALERT": "⚡",
-        "NOTICE": "ℹ",
-        "EXCEPTION": "❗",
-        "FATAL": "☠",
-        "DEFAULT": "•",
-    }
-
-    DEFAULT_COLORS = {
-        LogLevel.DEBUG: LogColor.DEBUG,
-        LogLevel.INFO: LogColor.INFO,
-        LogLevel.SUCCESS: LogColor.SUCCESS,
-        LogLevel.WARNING: LogColor.WARNING,
-        LogLevel.ERROR: LogColor.ERROR,
-    }
-
-    def __init__(
-        self,
-        log_file: Optional[str] = None,
-        log_level: LogLevel = LogLevel.DEBUG,
-        tag_width: int = 10,
-        icons: Optional[Dict[str, str]] = None,
-        colors: Optional[Dict[LogLevel, LogColor]] = None,
-        verbose: bool = True,
-    ):
-        """
-        Initialize the logger.
-
-        Args:
-            log_file: Optional file path for logging
-            log_level: Minimum log level to display
-            tag_width: Width for tag formatting
-            icons: Custom icons for different tags
-            colors: Custom colors for different log levels
-            verbose: Whether to output to console
-        """
-        self.log_file = log_file
-        self.log_level = log_level
-        self.tag_width = tag_width
-        self.icons = icons or self.DEFAULT_ICONS
-        self.colors = colors or self.DEFAULT_COLORS
-        self.verbose = verbose
-        self.console = Console()
-
-        # Create log file directory if needed
-        if log_file:
-            os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
-
-    def _format_tag(self, tag: str) -> str:
-        """Format a tag with consistent width."""
-        return f"[{tag}]".ljust(self.tag_width, ".")
-
-    def _get_icon(self, tag: str) -> str:
-        """Get the icon for a tag, defaulting to info icon if not found."""
-        return self.icons.get(tag, self.icons["INFO"])
-    
-    def _shorten(self, text, length, placeholder="..."):
-        """Truncate text in the middle if longer than length, or pad if shorter."""
-        if len(text) <= length:
-            return text.ljust(length)  # Pad with spaces to reach desired length
-        half = (length - len(placeholder)) // 2
-        shortened = text[:half] + placeholder + text[-half:]
-        return shortened.ljust(length)  # Also pad shortened text to consistent length
-
-    def _write_to_file(self, message: str):
-        """Write a message to the log file if configured."""
-        if self.log_file:
-            text = Text.from_markup(message)
-            plain_text = text.plain
-            timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
-            with open(self.log_file, "a", encoding="utf-8") as f:
-                f.write(f"[{timestamp}] {plain_text}\n")
-
-    def _log(
-        self,
-        level: LogLevel,
-        message: str,
-        tag: str,
-        params: Optional[Dict[str, Any]] = None,
-        colors: Optional[Dict[str, LogColor]] = None,
-        boxes: Optional[List[str]] = None,
-        base_color: Optional[LogColor] = None,
-        **kwargs,
-    ):
-        """
-        Core logging method that handles message formatting and output.
-
-        Args:
-            level: Log level for this message
-            message: Message template string
-            tag: Tag for the message
-            params: Parameters to format into the message
-            colors: Color overrides for specific parameters
-            boxes: Box overrides for specific parameters
-            base_color: Base color for the entire message
-        """
-        if level.value < self.log_level.value:
-            return
-
-        # avoid conflict with rich formatting
-        parsed_message = message.replace("[", "[[").replace("]", "]]")
-        if params:
-            # FIXME: If there are formatting strings in floating point format, 
-            # this may result in colors and boxes not being applied properly.
-            # such as {value:.2f}, the value is 0.23333 format it to 0.23,
-            # but we replace("0.23333", "[color]0.23333[/color]")
-            formatted_message = parsed_message.format(**params)
-            for key, value in params.items():
-                # value_str may discard `[` and `]`, so we need to replace it. 
-                value_str = str(value).replace("[", "[[").replace("]", "]]")
-                # check is need apply color
-                if colors and key in colors:
-                    color_str = f"[{colors[key]}]{value_str}[/{colors[key]}]"
-                    formatted_message = formatted_message.replace(value_str, color_str)
-                    value_str = color_str
-
-                # check is need apply box
-                if boxes and key in boxes:
-                    formatted_message = formatted_message.replace(value_str,
-                        create_box_message(value_str, type=str(level)))
-
-        else:
-            formatted_message = parsed_message
-
-        # Construct the full log line
-        color: LogColor = base_color or self.colors[level]
-        log_line = f"[{color}]{self._format_tag(tag)} {self._get_icon(tag)} {formatted_message} [/{color}]"
-
-        # Output to console if verbose
-        if self.verbose or kwargs.get("force_verbose", False):
-            self.console.print(log_line)
-
-        # Write to file if configured
-        self._write_to_file(log_line)
-
-    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
-        """Log a debug message."""
-        self._log(LogLevel.DEBUG, message, tag, **kwargs)
-
-    def info(self, message: str, tag: str = "INFO", **kwargs):
-        """Log an info message."""
-        self._log(LogLevel.INFO, message, tag, **kwargs)
-
-    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
-        """Log a success message."""
-        self._log(LogLevel.SUCCESS, message, tag, **kwargs)
-
-    def warning(self, message: str, tag: str = "WARNING", **kwargs):
-        """Log a warning message."""
-        self._log(LogLevel.WARNING, message, tag, **kwargs)
-        
-    def critical(self, message: str, tag: str = "CRITICAL", **kwargs):
-        """Log a critical message."""
-        self._log(LogLevel.ERROR, message, tag, **kwargs)
-    def exception(self, message: str, tag: str = "EXCEPTION", **kwargs):
-        """Log an exception message."""
-        self._log(LogLevel.ERROR, message, tag, **kwargs)
-    def fatal(self, message: str, tag: str = "FATAL", **kwargs):
-        """Log a fatal message."""
-        self._log(LogLevel.ERROR, message, tag, **kwargs)
-    def alert(self, message: str, tag: str = "ALERT", **kwargs):
-        """Log an alert message."""
-        self._log(LogLevel.ERROR, message, tag, **kwargs)
-    def notice(self, message: str, tag: str = "NOTICE", **kwargs):
-        """Log a notice message."""
-        self._log(LogLevel.INFO, message, tag, **kwargs)
-
-    def error(self, message: str, tag: str = "ERROR", **kwargs):
-        """Log an error message."""
-        self._log(LogLevel.ERROR, message, tag, **kwargs)
-
-    def url_status(
-        self,
-        url: str,
-        success: bool,
-        timing: float,
-        tag: str = "FETCH",
-        url_length: int = 100,
-    ):
-        """
-        Convenience method for logging URL fetch status.
-
-        Args:
-            url: The URL being processed
-            success: Whether the operation was successful
-            timing: Time taken for the operation
-            tag: Tag for the message
-            url_length: Maximum length for URL in log
-        """
-        decoded_url = unquote(url)
-        readable_url = self._shorten(decoded_url, url_length)
-        self._log(
-            level=LogLevel.SUCCESS if success else LogLevel.ERROR,
-            message="{url} | {status} | ⏱: {timing:.2f}s",
-            tag=tag,
-            params={
-                "url": readable_url,
-                "status": "✓" if success else "✗",
-                "timing": timing,
-            },
-            colors={
-                "status": LogColor.SUCCESS if success else LogColor.ERROR,
-                "timing": LogColor.WARNING,
-            },
-        )
-
-    def error_status(
-        self, url: str, error: str, tag: str = "ERROR", url_length: int = 50
-    ):
-        """
-        Convenience method for logging error status.
-
-        Args:
-            url: The URL being processed
-            error: Error message
-            tag: Tag for the message
-            url_length: Maximum length for URL in log
-        """
-        decoded_url = unquote(url)
-        readable_url = self._shorten(decoded_url, url_length)
-        self._log(
-            level=LogLevel.ERROR,
-            message="{url} | Error: {error}",
-            tag=tag,
-            params={"url": readable_url, "error": error},
-        )
-
-class AsyncFileLogger(AsyncLoggerBase):
-    """
-    File-only asynchronous logger that writes logs to a specified file.
-    """
-
-    def __init__(self, log_file: str):
-        """
-        Initialize the file logger.
-
-        Args:
-            log_file: File path for logging
-        """
-        self.log_file = log_file
-        os.makedirs(os.path.dirname(os.path.abspath(log_file)), exist_ok=True)
-
-    def _write_to_file(self, level: str, message: str, tag: str):
-        """Write a message to the log file."""
-        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
-        with open(self.log_file, "a", encoding="utf-8") as f:
-            f.write(f"[{timestamp}] [{level}] [{tag}] {message}\n")
-
-    def debug(self, message: str, tag: str = "DEBUG", **kwargs):
-        """Log a debug message to file."""
-        self._write_to_file("DEBUG", message, tag)
-
-    def info(self, message: str, tag: str = "INFO", **kwargs):
-        """Log an info message to file."""
-        self._write_to_file("INFO", message, tag)
-
-    def success(self, message: str, tag: str = "SUCCESS", **kwargs):
-        """Log a success message to file."""
-        self._write_to_file("SUCCESS", message, tag)
-
-    def warning(self, message: str, tag: str = "WARNING", **kwargs):
-        """Log a warning message to file."""
-        self._write_to_file("WARNING", message, tag)
-
-    def error(self, message: str, tag: str = "ERROR", **kwargs):
-        """Log an error message to file."""
-        self._write_to_file("ERROR", message, tag)
-
-    def url_status(self, url: str, success: bool, timing: float, tag: str = "FETCH", url_length: int = 100):
-        """Log URL fetch status to file."""
-        status = "SUCCESS" if success else "FAILED"
-        message = f"{url[:url_length]}... | Status: {status} | Time: {timing:.2f}s"
-        self._write_to_file("URL_STATUS", message, tag)
-
-    def error_status(self, url: str, error: str, tag: str = "ERROR", url_length: int = 100):
-        """Log error status to file."""
-        message = f"{url[:url_length]}... | Error: {error}"
-        self._write_to_file("ERROR", message, tag)

crawl4ai/browser_adapter.py

@@ -1,421 +0,0 @@
-# browser_adapter.py
-"""
-Browser adapter for Crawl4AI to support both Playwright and undetected browsers
-with minimal changes to existing codebase.
-"""
-
-from abc import ABC, abstractmethod
-from typing import List, Dict, Any, Optional, Callable
-import time
-import json
-
-# Import both, but use conditionally
-try:
-    from playwright.async_api import Page
-except ImportError:
-    Page = Any
-
-try:
-    from patchright.async_api import Page as UndetectedPage
-except ImportError:
-    UndetectedPage = Any
-
-
-class BrowserAdapter(ABC):
-    """Abstract adapter for browser-specific operations"""
-    
-    @abstractmethod
-    async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any:
-        """Execute JavaScript in the page"""
-        pass
-    
-    @abstractmethod
-    async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
-        """Setup console message capturing, returns handler function if needed"""
-        pass
-    
-    @abstractmethod
-    async def setup_error_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
-        """Setup error capturing, returns handler function if needed"""
-        pass
-    
-    @abstractmethod
-    async def retrieve_console_messages(self, page: Page) -> List[Dict]:
-        """Retrieve captured console messages (for undetected browsers)"""
-        pass
-    
-    @abstractmethod
-    async def cleanup_console_capture(self, page: Page, handle_console: Optional[Callable], handle_error: Optional[Callable]):
-        """Clean up console event listeners"""
-        pass
-    
-    @abstractmethod
-    def get_imports(self) -> tuple:
-        """Get the appropriate imports for this adapter"""
-        pass
-
-
-class PlaywrightAdapter(BrowserAdapter):
-    """Adapter for standard Playwright"""
-    
-    async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any:
-        """Standard Playwright evaluate"""
-        if arg is not None:
-            return await page.evaluate(expression, arg)
-        return await page.evaluate(expression)
-    
-    async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
-        """Setup console capture using Playwright's event system"""
-        def handle_console_capture(msg):
-            try:
-                message_type = "unknown"
-                try:
-                    message_type = msg.type
-                except:
-                    pass
-                    
-                message_text = "unknown"
-                try:
-                    message_text = msg.text
-                except:
-                    pass
-                    
-                entry = {
-                    "type": message_type,
-                    "text": message_text,
-                    "timestamp": time.time()
-                }
-                
-                captured_console.append(entry)
-                
-            except Exception as e:
-                captured_console.append({
-                    "type": "console_capture_error", 
-                    "error": str(e), 
-                    "timestamp": time.time()
-                })
-        
-        page.on("console", handle_console_capture)
-        return handle_console_capture
-    
-    async def setup_error_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
-        """Setup error capture using Playwright's event system"""
-        def handle_pageerror_capture(err):
-            try:
-                error_message = "Unknown error"
-                try:
-                    error_message = err.message
-                except:
-                    pass
-                    
-                error_stack = ""
-                try:
-                    error_stack = err.stack
-                except:
-                    pass
-                    
-                captured_console.append({
-                    "type": "error",
-                    "text": error_message,
-                    "stack": error_stack,
-                    "timestamp": time.time()
-                })
-            except Exception as e:
-                captured_console.append({
-                    "type": "pageerror_capture_error", 
-                    "error": str(e), 
-                    "timestamp": time.time()
-                })
-        
-        page.on("pageerror", handle_pageerror_capture)
-        return handle_pageerror_capture
-    
-    async def retrieve_console_messages(self, page: Page) -> List[Dict]:
-        """Not needed for Playwright - messages are captured via events"""
-        return []
-    
-    async def cleanup_console_capture(self, page: Page, handle_console: Optional[Callable], handle_error: Optional[Callable]):
-        """Remove event listeners"""
-        if handle_console:
-            page.remove_listener("console", handle_console)
-        if handle_error:
-            page.remove_listener("pageerror", handle_error)
-    
-    def get_imports(self) -> tuple:
-        """Return Playwright imports"""
-        from playwright.async_api import Page, Error
-        from playwright.async_api import TimeoutError as PlaywrightTimeoutError
-        return Page, Error, PlaywrightTimeoutError
-
-
-class StealthAdapter(BrowserAdapter):
-    """Adapter for Playwright with stealth features using playwright_stealth"""
-
-    def __init__(self):
-        self._console_script_injected = {}
-        self._stealth_available = self._check_stealth_availability()
-
-    def _check_stealth_availability(self) -> bool:
-        """Check if playwright_stealth is available and get the correct function"""
-        try:
-            from playwright_stealth import stealth_async
-            self._stealth_function = stealth_async
-            return True
-        except ImportError:
-            try:
-                from playwright_stealth import stealth_sync
-                self._stealth_function = stealth_sync
-                return True
-            except ImportError:
-                self._stealth_function = None
-                return False
-
-    async def apply_stealth(self, page: Page):
-        """Apply stealth to a page if available"""
-        if self._stealth_available and self._stealth_function:
-            try:
-                if hasattr(self._stealth_function, '__call__'):
-                    if 'async' in getattr(self._stealth_function, '__name__', ''):
-                        await self._stealth_function(page)
-                    else:
-                        self._stealth_function(page)
-            except Exception as e:
-                # Fail silently or log error depending on requirements
-                pass
-
-    async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any:
-        """Standard Playwright evaluate with stealth applied"""
-        if arg is not None:
-            return await page.evaluate(expression, arg)
-        return await page.evaluate(expression)
-
-    async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
-        """Setup console capture using Playwright's event system with stealth"""
-        # Apply stealth to the page first
-        await self.apply_stealth(page)
-
-        def handle_console_capture(msg):
-            try:
-                message_type = "unknown"
-                try:
-                    message_type = msg.type
-                except:
-                    pass
-
-                message_text = "unknown"
-                try:
-                    message_text = msg.text
-                except:
-                    pass
-
-                entry = {
-                    "type": message_type,
-                    "text": message_text,
-                    "timestamp": time.time()
-                }
-
-                captured_console.append(entry)
-
-            except Exception as e:
-                captured_console.append({
-                    "type": "console_capture_error",
-                    "error": str(e),
-                    "timestamp": time.time()
-                })
-
-        page.on("console", handle_console_capture)
-        return handle_console_capture
-
-    async def setup_error_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]:
-        """Setup error capture using Playwright's event system"""
-        def handle_pageerror_capture(err):
-            try:
-                error_message = "Unknown error"
-                try:
-                    error_message = err.message
-                except:
-                    pass
-
-                error_stack = ""
-                try:
-                    error_stack = err.stack
-                except:
-                    pass
-
-                captured_console.append({
-                    "type": "error",
-                    "text": error_message,
-                    "stack": error_stack,
-                    "timestamp": time.time()
-                })
-            except Exception as e:
-                captured_console.append({
-                    "type": "pageerror_capture_error",
-                    "error": str(e),
-                    "timestamp": time.time()
-                })
-
-        page.on("pageerror", handle_pageerror_capture)
-        return handle_pageerror_capture
-
-    async def retrieve_console_messages(self, page: Page) -> List[Dict]:
-        """Not needed for Playwright - messages are captured via events"""
-        return []
-
-    async def cleanup_console_capture(self, page: Page, handle_console: Optional[Callable], handle_error: Optional[Callable]):
-        """Remove event listeners"""
-        if handle_console:
-            page.remove_listener("console", handle_console)
-        if handle_error:
-            page.remove_listener("pageerror", handle_error)
-
-    def get_imports(self) -> tuple:
-        """Return Playwright imports"""
-        from playwright.async_api import Page, Error
-        from playwright.async_api import TimeoutError as PlaywrightTimeoutError
-        return Page, Error, PlaywrightTimeoutError
-
-
-class UndetectedAdapter(BrowserAdapter):
-    """Adapter for undetected browser automation with stealth features"""
-    
-    def __init__(self):
-        self._console_script_injected = {}
-    
-    async def evaluate(self, page: UndetectedPage, expression: str, arg: Any = None) -> Any:
-        """Undetected browser evaluate with isolated context"""
-        # For most evaluations, use isolated context for stealth
-        # Only use non-isolated when we need to access our injected console capture
-        isolated = not (
-            "__console" in expression or 
-            "__captured" in expression or
-            "__error" in expression or
-            "window.__" in expression
-        )
-        
-        if arg is not None:
-            return await page.evaluate(expression, arg, isolated_context=isolated)
-        return await page.evaluate(expression, isolated_context=isolated)
-    
-    async def setup_console_capture(self, page: UndetectedPage, captured_console: List[Dict]) -> Optional[Callable]:
-        """Setup console capture using JavaScript injection for undetected browsers"""
-        if not self._console_script_injected.get(page, False):
-            await page.add_init_script("""
-                // Initialize console capture
-                window.__capturedConsole = [];
-                window.__capturedErrors = [];
-                
-                // Store original console methods
-                const originalConsole = {};
-                ['log', 'info', 'warn', 'error', 'debug'].forEach(method => {
-                    originalConsole[method] = console[method];
-                    console[method] = function(...args) {
-                        try {
-                            window.__capturedConsole.push({
-                                type: method,
-                                text: args.map(arg => {
-                                    try {
-                                        if (typeof arg === 'object') {
-                                            return JSON.stringify(arg);
-                                        }
-                                        return String(arg);
-                                    } catch (e) {
-                                        return '[Object]';
-                                    }
-                                }).join(' '),
-                                timestamp: Date.now()
-                            });
-                        } catch (e) {
-                            // Fail silently to avoid detection
-                        }
-                        
-                        // Call original method
-                        originalConsole[method].apply(console, args);
-                    };
-                });
-            """)
-            self._console_script_injected[page] = True
-        
-        return None  # No handler function needed for undetected browser
-    
-    async def setup_error_capture(self, page: UndetectedPage, captured_console: List[Dict]) -> Optional[Callable]:
-        """Setup error capture using JavaScript injection for undetected browsers"""
-        if not self._console_script_injected.get(page, False):
-            await page.add_init_script("""
-                // Capture errors
-                window.addEventListener('error', (event) => {
-                    try {
-                        window.__capturedErrors.push({
-                            type: 'error',
-                            text: event.message,
-                            stack: event.error ? event.error.stack : '',
-                            filename: event.filename,
-                            lineno: event.lineno,
-                            colno: event.colno,
-                            timestamp: Date.now()
-                        });
-                    } catch (e) {
-                        // Fail silently
-                    }
-                });
-                
-                // Capture unhandled promise rejections
-                window.addEventListener('unhandledrejection', (event) => {
-                    try {
-                        window.__capturedErrors.push({
-                            type: 'unhandledrejection',
-                            text: event.reason ? String(event.reason) : 'Unhandled Promise Rejection',
-                            stack: event.reason && event.reason.stack ? event.reason.stack : '',
-                            timestamp: Date.now()
-                        });
-                    } catch (e) {
-                        // Fail silently
-                    }
-                });
-            """)
-            self._console_script_injected[page] = True
-        
-        return None  # No handler function needed for undetected browser
-    
-    async def retrieve_console_messages(self, page: UndetectedPage) -> List[Dict]:
-        """Retrieve captured console messages and errors from the page"""
-        messages = []
-        
-        try:
-            # Get console messages
-            console_messages = await page.evaluate(
-                "() => { const msgs = window.__capturedConsole || []; window.__capturedConsole = []; return msgs; }",
-                isolated_context=False
-            )
-            messages.extend(console_messages)
-            
-            # Get errors
-            errors = await page.evaluate(
-                "() => { const errs = window.__capturedErrors || []; window.__capturedErrors = []; return errs; }",
-                isolated_context=False
-            )
-            messages.extend(errors)
-            
-            # Convert timestamps from JS to Python format
-            for msg in messages:
-                if 'timestamp' in msg and isinstance(msg['timestamp'], (int, float)):
-                    msg['timestamp'] = msg['timestamp'] / 1000.0  # Convert from ms to seconds
-                    
-        except Exception:
-            # If retrieval fails, return empty list
-            pass
-        
-        return messages
-    
-    async def cleanup_console_capture(self, page: UndetectedPage, handle_console: Optional[Callable], handle_error: Optional[Callable]):
-        """Clean up for undetected browser - retrieve final messages"""
-        # For undetected browser, we don't have event listeners to remove
-        # but we should retrieve any final messages
-        final_messages = await self.retrieve_console_messages(page)
-        return final_messages
-    
-    def get_imports(self) -> tuple:
-        """Return undetected browser imports"""
-        from patchright.async_api import Page, Error
-        from patchright.async_api import TimeoutError as PlaywrightTimeoutError
-        return Page, Error, PlaywrightTimeoutError

crawl4ai/cache_context.py

@@ -1,117 +0,0 @@
-from enum import Enum
-
-
-class CacheMode(Enum):
-    """
-    Defines the caching behavior for web crawling operations.
-
-    Modes:
-    - ENABLED: Normal caching behavior (read and write)
-    - DISABLED: No caching at all
-    - READ_ONLY: Only read from cache, don't write
-    - WRITE_ONLY: Only write to cache, don't read
-    - BYPASS: Bypass cache for this operation
-    """
-
-    ENABLED = "enabled"
-    DISABLED = "disabled"
-    READ_ONLY = "read_only"
-    WRITE_ONLY = "write_only"
-    BYPASS = "bypass"
-
-
-class CacheContext:
-    """
-    Encapsulates cache-related decisions and URL handling.
-
-    This class centralizes all cache-related logic and URL type checking,
-    making the caching behavior more predictable and maintainable.
-
-    Attributes:
-        url (str): The URL being processed.
-        cache_mode (CacheMode): The cache mode for the current operation.
-        always_bypass (bool): If True, bypasses caching for this operation.
-        is_cacheable (bool): True if the URL is cacheable, False otherwise.
-        is_web_url (bool): True if the URL is a web URL, False otherwise.
-        is_local_file (bool): True if the URL is a local file, False otherwise.
-        is_raw_html (bool): True if the URL is raw HTML, False otherwise.
-        _url_display (str): The display name for the URL (web, local file, or raw HTML).
-    """
-
-    def __init__(self, url: str, cache_mode: CacheMode, always_bypass: bool = False):
-        """
-        Initializes the CacheContext with the provided URL and cache mode.
-
-        Args:
-            url (str): The URL being processed.
-            cache_mode (CacheMode): The cache mode for the current operation.
-            always_bypass (bool): If True, bypasses caching for this operation.
-        """
-        self.url = url
-        self.cache_mode = cache_mode
-        self.always_bypass = always_bypass
-        self.is_cacheable = url.startswith(("http://", "https://", "file://"))
-        self.is_web_url = url.startswith(("http://", "https://"))
-        self.is_local_file = url.startswith("file://")
-        self.is_raw_html = url.startswith("raw:")
-        self._url_display = url if not self.is_raw_html else "Raw HTML"
-
-    def should_read(self) -> bool:
-        """
-        Determines if cache should be read based on context.
-
-        How it works:
-        1. If always_bypass is True or is_cacheable is False, return False.
-        2. If cache_mode is ENABLED or READ_ONLY, return True.
-
-        Returns:
-            bool: True if cache should be read, False otherwise.
-        """
-        if self.always_bypass or not self.is_cacheable:
-            return False
-        return self.cache_mode in [CacheMode.ENABLED, CacheMode.READ_ONLY]
-
-    def should_write(self) -> bool:
-        """
-        Determines if cache should be written based on context.
-
-        How it works:
-        1. If always_bypass is True or is_cacheable is False, return False.
-        2. If cache_mode is ENABLED or WRITE_ONLY, return True.
-
-        Returns:
-            bool: True if cache should be written, False otherwise.
-        """
-        if self.always_bypass or not self.is_cacheable:
-            return False
-        return self.cache_mode in [CacheMode.ENABLED, CacheMode.WRITE_ONLY]
-
-    @property
-    def display_url(self) -> str:
-        """Returns the URL in display format."""
-        return self._url_display
-
-
-def _legacy_to_cache_mode(
-    disable_cache: bool = False,
-    bypass_cache: bool = False,
-    no_cache_read: bool = False,
-    no_cache_write: bool = False,
-) -> CacheMode:
-    """
-    Converts legacy cache parameters to the new CacheMode enum.
-
-    This is an internal function to help transition from the old boolean flags
-    to the new CacheMode system.
-    """
-    if disable_cache:
-        return CacheMode.DISABLED
-    if bypass_cache:
-        return CacheMode.BYPASS
-    if no_cache_read and no_cache_write:
-        return CacheMode.DISABLED
-    if no_cache_read:
-        return CacheMode.WRITE_ONLY
-    if no_cache_write:
-        return CacheMode.READ_ONLY
-    return CacheMode.ENABLED

crawl4ai/chunking_strategy.py

@@ -3,52 +3,23 @@ import re
 from collections import Counter
 import string
 from .model_loader import load_nltk_punkt
+from .utils import *
 
 # Define the abstract base class for chunking strategies
 class ChunkingStrategy(ABC):
-    """
-    Abstract base class for chunking strategies.
-    """
-
+    
     @abstractmethod
     def chunk(self, text: str) -> list:
         """
         Abstract method to chunk the given text.
-
-        Args:
-            text (str): The text to chunk.
-
-        Returns:
-            list: A list of chunks.
         """
         pass
-
-
-# Create an identity chunking strategy f(x) = [x]
-class IdentityChunking(ChunkingStrategy):
-    """
-    Chunking strategy that returns the input text as a single chunk.
-    """
-
-    def chunk(self, text: str) -> list:
-        return [text]
-
-
+    
 # Regex-based chunking
 class RegexChunking(ChunkingStrategy):
-    """
-    Chunking strategy that splits text based on regular expression patterns.
-    """
-
     def __init__(self, patterns=None, **kwargs):
-        """
-        Initialize the RegexChunking object.
-
-        Args:
-            patterns (list): A list of regular expression patterns to split text.
-        """
         if patterns is None:
-            patterns = [r"\n\n"]  # Default split pattern
+            patterns = [r'\n\n']  # Default split pattern
         self.patterns = patterns
 
     def chunk(self, text: str) -> list:
@@ -59,20 +30,12 @@ class RegexChunking(ChunkingStrategy):
                 new_paragraphs.extend(re.split(pattern, paragraph))
             paragraphs = new_paragraphs
         return paragraphs
-
-
-# NLP-based sentence chunking
+    
+# NLP-based sentence chunking 
 class NlpSentenceChunking(ChunkingStrategy):
-    """
-    Chunking strategy that splits text into sentences using NLTK's sentence tokenizer.
-    """
-
     def __init__(self, **kwargs):
-        """
-        Initialize the NlpSentenceChunking object.
-        """
-        from crawl4ai.le.legacy.model_loader import load_nltk_punkt
         load_nltk_punkt()
+        pass
 
     def chunk(self, text: str) -> list:
         # Improved regex for sentence splitting
@@ -80,34 +43,18 @@ class NlpSentenceChunking(ChunkingStrategy):
         #     r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<![A-Z][A-Z]\.)(?<![A-Za-z]\.)(?<=\.|\?|\!|\n)\s'
         # )
         # sentences = sentence_endings.split(text)
-        # sens =  [sent.strip() for sent in sentences if sent]
+        # sens =  [sent.strip() for sent in sentences if sent]            
         from nltk.tokenize import sent_tokenize
-
         sentences = sent_tokenize(text)
-        sens = [sent.strip() for sent in sentences]
-
+        sens =  [sent.strip() for sent in sentences]        
+        
         return list(set(sens))
-
-
+    
 # Topic-based segmentation using TextTiling
 class TopicSegmentationChunking(ChunkingStrategy):
-    """
-    Chunking strategy that segments text into topics using NLTK's TextTilingTokenizer.
-
-    How it works:
-    1. Segment the text into topics using TextTilingTokenizer
-    2. Extract keywords for each topic segment
-    """
-
+    
     def __init__(self, num_keywords=3, **kwargs):
-        """
-        Initialize the TopicSegmentationChunking object.
-
-        Args:
-            num_keywords (int): The number of keywords to extract for each topic segment.
-        """
         import nltk as nl
-
         self.tokenizer = nl.tokenize.TextTilingTokenizer()
         self.num_keywords = num_keywords
 
@@ -119,14 +66,8 @@ class TopicSegmentationChunking(ChunkingStrategy):
     def extract_keywords(self, text: str) -> list:
         # Tokenize and remove stopwords and punctuation
         import nltk as nl
-
         tokens = nl.toknize.word_tokenize(text)
-        tokens = [
-            token.lower()
-            for token in tokens
-            if token not in nl.corpus.stopwords.words("english")
-            and token not in string.punctuation
-        ]
+        tokens = [token.lower() for token in tokens if token not in nl.corpus.stopwords.words('english') and token not in string.punctuation]
 
         # Calculate frequency distribution
         freq_dist = Counter(tokens)
@@ -137,120 +78,29 @@ class TopicSegmentationChunking(ChunkingStrategy):
         # Segment the text into topics
         segments = self.chunk(text)
         # Extract keywords for each topic segment
-        segments_with_topics = [
-            (segment, self.extract_keywords(segment)) for segment in segments
-        ]
+        segments_with_topics = [(segment, self.extract_keywords(segment)) for segment in segments]
         return segments_with_topics
-
-
+    
 # Fixed-length word chunks
 class FixedLengthWordChunking(ChunkingStrategy):
-    """
-    Chunking strategy that splits text into fixed-length word chunks.
-
-    How it works:
-    1. Split the text into words
-    2. Create chunks of fixed length
-    3. Return the list of chunks
-    """
-
     def __init__(self, chunk_size=100, **kwargs):
-        """
-        Initialize the fixed-length word chunking strategy with the given chunk size.
-
-        Args:
-            chunk_size (int): The size of each chunk in words.
-        """
         self.chunk_size = chunk_size
 
     def chunk(self, text: str) -> list:
         words = text.split()
-        return [
-            " ".join(words[i : i + self.chunk_size])
-            for i in range(0, len(words), self.chunk_size)
-        ]
-
-
+        return [' '.join(words[i:i + self.chunk_size]) for i in range(0, len(words), self.chunk_size)]
+    
 # Sliding window chunking
 class SlidingWindowChunking(ChunkingStrategy):
-    """
-    Chunking strategy that splits text into overlapping word chunks.
-
-    How it works:
-    1. Split the text into words
-    2. Create chunks of fixed length
-    3. Return the list of chunks
-    """
-
     def __init__(self, window_size=100, step=50, **kwargs):
-        """
-        Initialize the sliding window chunking strategy with the given window size and
-        step size.
-
-        Args:
-            window_size (int): The size of the sliding window in words.
-            step (int): The step size for sliding the window in words.
-        """
         self.window_size = window_size
         self.step = step
 
     def chunk(self, text: str) -> list:
         words = text.split()
         chunks = []
-
-        if len(words) <= self.window_size:
-            return [text]
-
-        for i in range(0, len(words) - self.window_size + 1, self.step):
-            chunk = " ".join(words[i : i + self.window_size])
-            chunks.append(chunk)
-
-        # Handle the last chunk if it doesn't align perfectly
-        if i + self.window_size < len(words):
-            chunks.append(" ".join(words[-self.window_size :]))
-
+        for i in range(0, len(words), self.step):
+            chunks.append(' '.join(words[i:i + self.window_size]))
         return chunks
+    
 
-
-class OverlappingWindowChunking(ChunkingStrategy):
-    """
-    Chunking strategy that splits text into overlapping word chunks.
-
-    How it works:
-    1. Split the text into words using whitespace
-    2. Create chunks of fixed length equal to the window size
-    3. Slide the window by the overlap size
-    4. Return the list of chunks
-    """
-
-    def __init__(self, window_size=1000, overlap=100, **kwargs):
-        """
-        Initialize the overlapping window chunking strategy with the given window size and
-        overlap size.
-
-        Args:
-            window_size (int): The size of the window in words.
-            overlap (int): The size of the overlap between consecutive chunks in words.
-        """
-        self.window_size = window_size
-        self.overlap = overlap
-
-    def chunk(self, text: str) -> list:
-        words = text.split()
-        chunks = []
-
-        if len(words) <= self.window_size:
-            return [text]
-
-        start = 0
-        while start < len(words):
-            end = start + self.window_size
-            chunk = " ".join(words[start:end])
-            chunks.append(chunk)
-
-            if end >= len(words):
-                break
-
-            start = end - self.overlap
-
-        return chunks

crawl4ai/components/crawler_monitor.py

@@ -1,837 +0,0 @@
-import time
-import uuid
-import threading
-import psutil
-from datetime import datetime, timedelta
-from typing import Dict, Optional, List
-import threading
-from rich.console import Console
-from rich.layout import Layout
-from rich.panel import Panel
-from rich.table import Table
-from rich.text import Text
-from rich.live import Live
-from rich import box
-from ..models import CrawlStatus
-
-class TerminalUI:
-    """Terminal user interface for CrawlerMonitor using rich library."""
-    
-    def __init__(self, refresh_rate: float = 1.0, max_width: int = 120):
-        """
-        Initialize the terminal UI.
-        
-        Args:
-            refresh_rate: How often to refresh the UI (in seconds)
-            max_width: Maximum width of the UI in characters
-        """
-        self.console = Console(width=max_width)
-        self.layout = Layout()
-        self.refresh_rate = refresh_rate
-        self.stop_event = threading.Event()
-        self.ui_thread = None
-        self.monitor = None  # Will be set by CrawlerMonitor
-        self.max_width = max_width
-        
-        # Setup layout - vertical layout (top to bottom)
-        self.layout.split(
-            Layout(name="header", size=3),
-            Layout(name="pipeline_status", size=10),
-            Layout(name="task_details", ratio=1),
-            Layout(name="footer", size=3)  # Increased footer size to fit all content
-        )
-        
-    def start(self, monitor):
-        """Start the UI thread."""
-        self.monitor = monitor
-        self.stop_event.clear()
-        self.ui_thread = threading.Thread(target=self._ui_loop)
-        self.ui_thread.daemon = True
-        self.ui_thread.start()
-        
-    def stop(self):
-        """Stop the UI thread."""
-        if self.ui_thread and self.ui_thread.is_alive():
-            self.stop_event.set()
-            # Only try to join if we're not in the UI thread
-            # This prevents "cannot join current thread" errors
-            if threading.current_thread() != self.ui_thread:
-                self.ui_thread.join(timeout=5.0)
-    
-    def _ui_loop(self):
-        """Main UI rendering loop."""
-        import sys
-        import select
-        import termios
-        import tty
-        
-        # Setup terminal for non-blocking input
-        old_settings = termios.tcgetattr(sys.stdin)
-        try:
-            tty.setcbreak(sys.stdin.fileno())
-            
-            # Use Live display to render the UI
-            with Live(self.layout, refresh_per_second=1/self.refresh_rate, screen=True) as live:
-                self.live = live  # Store the live display for updates
-                
-                # Main UI loop
-                while not self.stop_event.is_set():
-                    self._update_display()
-                    
-                    # Check for key press (non-blocking)
-                    if select.select([sys.stdin], [], [], 0)[0]:
-                        key = sys.stdin.read(1)
-                        # Check for 'q' to quit
-                        if key == 'q':
-                            # Signal stop but don't call monitor.stop() from UI thread
-                            # as it would cause the thread to try to join itself
-                            self.stop_event.set()
-                            self.monitor.is_running = False
-                            break
-                    
-                    time.sleep(self.refresh_rate)
-                    
-                    # Just check if the monitor was stopped
-                    if not self.monitor.is_running:
-                        break
-        finally:
-            # Restore terminal settings
-            termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings)
-    
-    def _update_display(self):
-        """Update the terminal display with current statistics."""
-        if not self.monitor:
-            return
-            
-        # Update crawler status panel
-        self.layout["header"].update(self._create_status_panel())
-        
-        # Update pipeline status panel and task details panel
-        self.layout["pipeline_status"].update(self._create_pipeline_panel())
-        self.layout["task_details"].update(self._create_task_details_panel())
-        
-        # Update footer
-        self.layout["footer"].update(self._create_footer())
-    
-    def _create_status_panel(self) -> Panel:
-        """Create the crawler status panel."""
-        summary = self.monitor.get_summary()
-        
-        # Format memory status with icon
-        memory_status = self.monitor.get_memory_status()
-        memory_icon = "🟢"  # Default NORMAL
-        if memory_status == "PRESSURE":
-            memory_icon = "🟠"
-        elif memory_status == "CRITICAL":
-            memory_icon = "🔴"
-        
-        # Get current memory usage
-        current_memory = psutil.Process().memory_info().rss / (1024 * 1024)  # MB
-        memory_percent = (current_memory / psutil.virtual_memory().total) * 100
-        
-        # Format runtime
-        runtime = self.monitor._format_time(time.time() - self.monitor.start_time if self.monitor.start_time else 0)
-        
-        # Create the status text
-        status_text = Text()
-        status_text.append(f"Web Crawler Dashboard | Runtime: {runtime} | Memory: {memory_percent:.1f}% {memory_icon}\n")
-        status_text.append(f"Status: {memory_status} | URLs: {summary['urls_completed']}/{summary['urls_total']} | ")
-        status_text.append(f"Peak Mem: {summary['peak_memory_percent']:.1f}% at {self.monitor._format_time(summary['peak_memory_time'])}")
-        
-        return Panel(status_text, title="Crawler Status", border_style="blue")
-    
-    def _create_pipeline_panel(self) -> Panel:
-        """Create the pipeline status panel."""
-        summary = self.monitor.get_summary()
-        queue_stats = self.monitor.get_queue_stats()
-        
-        # Create a table for status counts
-        table = Table(show_header=True, box=None)
-        table.add_column("Status", style="cyan")
-        table.add_column("Count", justify="right")
-        table.add_column("Percentage", justify="right")
-        table.add_column("Stat", style="cyan")
-        table.add_column("Value", justify="right")
-        
-        # Calculate overall progress
-        progress = f"{summary['urls_completed']}/{summary['urls_total']}"
-        progress_percent = f"{summary['completion_percentage']:.1f}%"
-        
-        # Add rows for each status
-        table.add_row(
-            "Overall Progress", 
-            progress, 
-            progress_percent,
-            "Est. Completion", 
-            summary.get('estimated_completion_time', "N/A")
-        )
-        
-        # Add rows for each status
-        status_counts = summary['status_counts']
-        total = summary['urls_total'] or 1  # Avoid division by zero
-        
-        # Status rows
-        table.add_row(
-            "Completed", 
-            str(status_counts.get(CrawlStatus.COMPLETED.name, 0)),
-            f"{status_counts.get(CrawlStatus.COMPLETED.name, 0) / total * 100:.1f}%",
-            "Avg. Time/URL",
-            f"{summary.get('avg_task_duration', 0):.2f}s"
-        )
-        
-        table.add_row(
-            "Failed", 
-            str(status_counts.get(CrawlStatus.FAILED.name, 0)),
-            f"{status_counts.get(CrawlStatus.FAILED.name, 0) / total * 100:.1f}%",
-            "Concurrent Tasks",
-            str(status_counts.get(CrawlStatus.IN_PROGRESS.name, 0))
-        )
-        
-        table.add_row(
-            "In Progress", 
-            str(status_counts.get(CrawlStatus.IN_PROGRESS.name, 0)),
-            f"{status_counts.get(CrawlStatus.IN_PROGRESS.name, 0) / total * 100:.1f}%",
-            "Queue Size",
-            str(queue_stats['total_queued'])
-        )
-        
-        table.add_row(
-            "Queued", 
-            str(status_counts.get(CrawlStatus.QUEUED.name, 0)),
-            f"{status_counts.get(CrawlStatus.QUEUED.name, 0) / total * 100:.1f}%",
-            "Max Wait Time",
-            f"{queue_stats['highest_wait_time']:.1f}s"
-        )
-        
-        # Requeued is a special case as it's not a status
-        requeued_count = summary.get('requeued_count', 0)
-        table.add_row(
-            "Requeued", 
-            str(requeued_count),
-            f"{summary.get('requeue_rate', 0):.1f}%",
-            "Avg Wait Time",
-            f"{queue_stats['avg_wait_time']:.1f}s"
-        )
-        
-        # Add empty row for spacing
-        table.add_row(
-            "", 
-            "",
-            "",
-            "Requeue Rate",
-            f"{summary.get('requeue_rate', 0):.1f}%"
-        )
-        
-        return Panel(table, title="Pipeline Status", border_style="green")
-    
-    def _create_task_details_panel(self) -> Panel:
-        """Create the task details panel."""
-        # Create a table for task details
-        table = Table(show_header=True, expand=True)
-        table.add_column("Task ID", style="cyan", no_wrap=True, width=10)
-        table.add_column("URL", style="blue", ratio=3)
-        table.add_column("Status", style="green", width=15)
-        table.add_column("Memory", justify="right", width=8)
-        table.add_column("Peak", justify="right", width=8)
-        table.add_column("Duration", justify="right", width=10)
-        
-        # Get all task stats
-        task_stats = self.monitor.get_all_task_stats()
-        
-        # Add summary row
-        active_tasks = sum(1 for stats in task_stats.values() 
-                          if stats['status'] == CrawlStatus.IN_PROGRESS.name)
-        
-        total_memory = sum(stats['memory_usage'] for stats in task_stats.values())
-        total_peak = sum(stats['peak_memory'] for stats in task_stats.values())
-        
-        # Summary row with separators
-        table.add_row(
-            "SUMMARY", 
-            f"Total: {len(task_stats)}", 
-            f"Active: {active_tasks}",
-            f"{total_memory:.1f}",
-            f"{total_peak:.1f}",
-            "N/A"
-        )
-        
-        # Add a separator
-        table.add_row("—" * 10, "—" * 20, "—" * 10, "—" * 8, "—" * 8, "—" * 10)
-        
-        # Status icons
-        status_icons = {
-            CrawlStatus.QUEUED.name: "⏳",
-            CrawlStatus.IN_PROGRESS.name: "🔄",
-            CrawlStatus.COMPLETED.name: "✅",
-            CrawlStatus.FAILED.name: "❌"
-        }
-        
-        # Calculate how many rows we can display based on available space
-        # We can display more rows now that we have a dedicated panel
-        display_count = min(len(task_stats), 20)  # Display up to 20 tasks
-        
-        # Add rows for each task
-        for task_id, stats in sorted(
-            list(task_stats.items())[:display_count],
-            # Sort: 1. IN_PROGRESS first, 2. QUEUED, 3. COMPLETED/FAILED by recency
-            key=lambda x: (
-                0 if x[1]['status'] == CrawlStatus.IN_PROGRESS.name else 
-                1 if x[1]['status'] == CrawlStatus.QUEUED.name else 
-                2,
-                -1 * (x[1].get('end_time', 0) or 0)  # Most recent first
-            )
-        ):
-            # Truncate task_id and URL for display
-            short_id = task_id[:8]
-            url = stats['url']
-            if len(url) > 50:  # Allow longer URLs in the dedicated panel
-                url = url[:47] + "..."
-                
-            # Format status with icon
-            status = f"{status_icons.get(stats['status'], '?')} {stats['status']}"
-            
-            # Add row
-            table.add_row(
-                short_id,
-                url,
-                status,
-                f"{stats['memory_usage']:.1f}",
-                f"{stats['peak_memory']:.1f}",
-                stats['duration'] if 'duration' in stats else "0:00"
-            )
-        
-        return Panel(table, title="Task Details", border_style="yellow")
-    
-    def _create_footer(self) -> Panel:
-        """Create the footer panel."""
-        from rich.columns import Columns
-        from rich.align import Align
-        
-        memory_status = self.monitor.get_memory_status()
-        memory_icon = "🟢"  # Default NORMAL
-        if memory_status == "PRESSURE":
-            memory_icon = "🟠"
-        elif memory_status == "CRITICAL":
-            memory_icon = "🔴"
-        
-        # Left section - memory status
-        left_text = Text()
-        left_text.append("Memory Status: ", style="bold")
-        status_style = "green" if memory_status == "NORMAL" else "yellow" if memory_status == "PRESSURE" else "red bold"
-        left_text.append(f"{memory_icon} {memory_status}", style=status_style)
-        
-        # Center section - copyright
-        center_text = Text("© Crawl4AI 2025 | Made by UnclecCode", style="cyan italic")
-        
-        # Right section - quit instruction
-        right_text = Text()
-        right_text.append("Press ", style="bold")
-        right_text.append("q", style="white on blue")
-        right_text.append(" to quit", style="bold")
-        
-        # Create columns with the three sections
-        footer_content = Columns(
-            [
-                Align.left(left_text),
-                Align.center(center_text),
-                Align.right(right_text)
-            ],
-            expand=True
-        )
-        
-        # Create a more visible footer panel
-        return Panel(
-            footer_content,
-            border_style="white",
-            padding=(0, 1)  # Add padding for better visibility
-        )
-
-
-class CrawlerMonitor:
-    """
-    Comprehensive monitoring and visualization system for tracking web crawler operations in real-time.
-    Provides a terminal-based dashboard that displays task statuses, memory usage, queue statistics,
-    and performance metrics.
-    """
-    
-    def __init__(
-        self,
-        urls_total: int = 0,
-        refresh_rate: float = 1.0,
-        enable_ui: bool = True,
-        max_width: int = 120
-    ):
-        """
-        Initialize the CrawlerMonitor.
-        
-        Args:
-            urls_total: Total number of URLs to be crawled
-            refresh_rate: How often to refresh the UI (in seconds)
-            enable_ui: Whether to display the terminal UI
-            max_width: Maximum width of the UI in characters
-        """
-        # Core monitoring attributes
-        self.stats = {}  # Task ID -> stats dict
-        self.memory_status = "NORMAL"
-        self.start_time = None
-        self.end_time = None
-        self.is_running = False
-        self.queue_stats = {
-            "total_queued": 0,
-            "highest_wait_time": 0.0,
-            "avg_wait_time": 0.0
-        }
-        self.urls_total = urls_total
-        self.urls_completed = 0
-        self.peak_memory_percent = 0.0
-        self.peak_memory_time = 0.0
-        
-        # Status counts
-        self.status_counts = {
-            CrawlStatus.QUEUED.name: 0,
-            CrawlStatus.IN_PROGRESS.name: 0,
-            CrawlStatus.COMPLETED.name: 0,
-            CrawlStatus.FAILED.name: 0
-        }
-        
-        # Requeue tracking
-        self.requeued_count = 0
-        
-        # Thread-safety
-        self._lock = threading.RLock()
-        
-        # Terminal UI
-        self.enable_ui = enable_ui
-        self.terminal_ui = TerminalUI(
-            refresh_rate=refresh_rate, 
-            max_width=max_width
-        ) if enable_ui else None
-    
-    def start(self):
-        """
-        Start the monitoring session.
-        
-        - Initializes the start_time
-        - Sets is_running to True
-        - Starts the terminal UI if enabled
-        """
-        with self._lock:
-            self.start_time = time.time()
-            self.is_running = True
-            
-            # Start the terminal UI
-            if self.enable_ui and self.terminal_ui:
-                self.terminal_ui.start(self)
-    
-    def stop(self):
-        """
-        Stop the monitoring session.
-        
-        - Records end_time
-        - Sets is_running to False
-        - Stops the terminal UI
-        - Generates final summary statistics
-        """
-        with self._lock:
-            self.end_time = time.time()
-            self.is_running = False
-            
-            # Stop the terminal UI
-            if self.enable_ui and self.terminal_ui:
-                self.terminal_ui.stop()
-    
-    def add_task(self, task_id: str, url: str):
-        """
-        Register a new task with the monitor.
-        
-        Args:
-            task_id: Unique identifier for the task
-            url: URL being crawled
-            
-        The task is initialized with:
-            - status: QUEUED
-            - url: The URL to crawl
-            - enqueue_time: Current time
-            - memory_usage: 0
-            - peak_memory: 0
-            - wait_time: 0
-            - retry_count: 0
-        """
-        with self._lock:
-            self.stats[task_id] = {
-                "task_id": task_id,
-                "url": url,
-                "status": CrawlStatus.QUEUED.name,
-                "enqueue_time": time.time(),
-                "start_time": None,
-                "end_time": None,
-                "memory_usage": 0.0,
-                "peak_memory": 0.0,
-                "error_message": "",
-                "wait_time": 0.0,
-                "retry_count": 0,
-                "duration": "0:00",
-                "counted_requeue": False
-            }
-            
-            # Update status counts
-            self.status_counts[CrawlStatus.QUEUED.name] += 1
-    
-    def update_task(
-        self, 
-        task_id: str, 
-        status: Optional[CrawlStatus] = None,
-        start_time: Optional[float] = None,
-        end_time: Optional[float] = None,
-        memory_usage: Optional[float] = None,
-        peak_memory: Optional[float] = None,
-        error_message: Optional[str] = None,
-        retry_count: Optional[int] = None,
-        wait_time: Optional[float] = None
-    ):
-        """
-        Update statistics for a specific task.
-        
-        Args:
-            task_id: Unique identifier for the task
-            status: New status (QUEUED, IN_PROGRESS, COMPLETED, FAILED)
-            start_time: When task execution started
-            end_time: When task execution ended
-            memory_usage: Current memory usage in MB
-            peak_memory: Maximum memory usage in MB
-            error_message: Error description if failed
-            retry_count: Number of retry attempts
-            wait_time: Time spent in queue
-            
-        Updates task statistics and updates status counts.
-        If status changes, decrements old status count and 
-        increments new status count.
-        """
-        with self._lock:
-            # Check if task exists
-            if task_id not in self.stats:
-                return
-            
-            task_stats = self.stats[task_id]
-            
-            # Update status counts if status is changing
-            old_status = task_stats["status"]
-            if status and status.name != old_status:
-                self.status_counts[old_status] -= 1
-                self.status_counts[status.name] += 1
-                
-                # Track completion
-                if status == CrawlStatus.COMPLETED:
-                    self.urls_completed += 1
-                
-                # Track requeues
-                if old_status in [CrawlStatus.COMPLETED.name, CrawlStatus.FAILED.name] and not task_stats.get("counted_requeue", False):
-                    self.requeued_count += 1
-                    task_stats["counted_requeue"] = True
-            
-            # Update task statistics
-            if status:
-                task_stats["status"] = status.name
-            if start_time is not None:
-                task_stats["start_time"] = start_time
-            if end_time is not None:
-                task_stats["end_time"] = end_time
-            if memory_usage is not None:
-                task_stats["memory_usage"] = memory_usage
-                
-                # Update peak memory if necessary
-                current_percent = (memory_usage / psutil.virtual_memory().total) * 100
-                if current_percent > self.peak_memory_percent:
-                    self.peak_memory_percent = current_percent
-                    self.peak_memory_time = time.time()
-                
-            if peak_memory is not None:
-                task_stats["peak_memory"] = peak_memory
-            if error_message is not None:
-                task_stats["error_message"] = error_message
-            if retry_count is not None:
-                task_stats["retry_count"] = retry_count
-            if wait_time is not None:
-                task_stats["wait_time"] = wait_time
-            
-            # Calculate duration
-            if task_stats["start_time"]:
-                end = task_stats["end_time"] or time.time()
-                duration = end - task_stats["start_time"]
-                task_stats["duration"] = self._format_time(duration)
-    
-    def update_memory_status(self, status: str):
-        """
-        Update the current memory status.
-        
-        Args:
-            status: Memory status (NORMAL, PRESSURE, CRITICAL, or custom)
-            
-        Also updates the UI to reflect the new status.
-        """
-        with self._lock:
-            self.memory_status = status
-    
-    def update_queue_statistics(
-        self,
-        total_queued: int,
-        highest_wait_time: float,
-        avg_wait_time: float
-    ):
-        """
-        Update statistics related to the task queue.
-        
-        Args:
-            total_queued: Number of tasks currently in queue
-            highest_wait_time: Longest wait time of any queued task
-            avg_wait_time: Average wait time across all queued tasks
-        """
-        with self._lock:
-            self.queue_stats = {
-                "total_queued": total_queued,
-                "highest_wait_time": highest_wait_time,
-                "avg_wait_time": avg_wait_time
-            }
-    
-    def get_task_stats(self, task_id: str) -> Dict:
-        """
-        Get statistics for a specific task.
-        
-        Args:
-            task_id: Unique identifier for the task
-            
-        Returns:
-            Dictionary containing all task statistics
-        """
-        with self._lock:
-            return self.stats.get(task_id, {}).copy()
-    
-    def get_all_task_stats(self) -> Dict[str, Dict]:
-        """
-        Get statistics for all tasks.
-        
-        Returns:
-            Dictionary mapping task_ids to their statistics
-        """
-        with self._lock:
-            return self.stats.copy()
-    
-    def get_memory_status(self) -> str:
-        """
-        Get the current memory status.
-        
-        Returns:
-            Current memory status string
-        """
-        with self._lock:
-            return self.memory_status
-    
-    def get_queue_stats(self) -> Dict:
-        """
-        Get current queue statistics.
-        
-        Returns:
-            Dictionary with queue statistics including:
-            - total_queued: Number of tasks in queue
-            - highest_wait_time: Longest wait time
-            - avg_wait_time: Average wait time
-        """
-        with self._lock:
-            return self.queue_stats.copy()
-    
-    def get_summary(self) -> Dict:
-        """
-        Get a summary of all crawler statistics.
-        
-        Returns:
-            Dictionary containing:
-            - runtime: Total runtime in seconds
-            - urls_total: Total URLs to process
-            - urls_completed: Number of completed URLs
-            - completion_percentage: Percentage complete
-            - status_counts: Count of tasks in each status
-            - memory_status: Current memory status
-            - peak_memory_percent: Highest memory usage
-            - peak_memory_time: When peak memory occurred
-            - avg_task_duration: Average task processing time
-            - estimated_completion_time: Projected finish time
-            - requeue_rate: Percentage of tasks requeued
-        """
-        with self._lock:
-            # Calculate runtime
-            current_time = time.time()
-            runtime = current_time - (self.start_time or current_time)
-            
-            # Calculate completion percentage
-            completion_percentage = 0
-            if self.urls_total > 0:
-                completion_percentage = (self.urls_completed / self.urls_total) * 100
-            
-            # Calculate average task duration for completed tasks
-            completed_tasks = [
-                task for task in self.stats.values() 
-                if task["status"] == CrawlStatus.COMPLETED.name and task.get("start_time") and task.get("end_time")
-            ]
-            
-            avg_task_duration = 0
-            if completed_tasks:
-                total_duration = sum(task["end_time"] - task["start_time"] for task in completed_tasks)
-                avg_task_duration = total_duration / len(completed_tasks)
-            
-            # Calculate requeue rate
-            requeue_rate = 0
-            if len(self.stats) > 0:
-                requeue_rate = (self.requeued_count / len(self.stats)) * 100
-            
-            # Calculate estimated completion time
-            estimated_completion_time = "N/A"
-            if avg_task_duration > 0 and self.urls_total > 0 and self.urls_completed > 0:
-                remaining_tasks = self.urls_total - self.urls_completed
-                estimated_seconds = remaining_tasks * avg_task_duration
-                estimated_completion_time = self._format_time(estimated_seconds)
-            
-            return {
-                "runtime": runtime,
-                "urls_total": self.urls_total,
-                "urls_completed": self.urls_completed,
-                "completion_percentage": completion_percentage,
-                "status_counts": self.status_counts.copy(),
-                "memory_status": self.memory_status,
-                "peak_memory_percent": self.peak_memory_percent,
-                "peak_memory_time": self.peak_memory_time,
-                "avg_task_duration": avg_task_duration,
-                "estimated_completion_time": estimated_completion_time,
-                "requeue_rate": requeue_rate,
-                "requeued_count": self.requeued_count
-            }
-    
-    def render(self):
-        """
-        Render the terminal UI.
-        
-        This is the main UI rendering loop that:
-        1. Updates all statistics
-        2. Formats the display
-        3. Renders the ASCII interface
-        4. Handles keyboard input
-        
-        Note: The actual rendering is handled by the TerminalUI class
-        which uses the rich library's Live display.
-        """
-        if self.enable_ui and self.terminal_ui:
-            # Force an update of the UI
-            if hasattr(self.terminal_ui, '_update_display'):
-                self.terminal_ui._update_display()
-    
-    def _format_time(self, seconds: float) -> str:
-        """
-        Format time in hours:minutes:seconds.
-        
-        Args:
-            seconds: Time in seconds
-            
-        Returns:
-            Formatted time string (e.g., "1:23:45")
-        """
-        delta = timedelta(seconds=int(seconds))
-        hours, remainder = divmod(delta.seconds, 3600)
-        minutes, seconds = divmod(remainder, 60)
-        
-        if hours > 0:
-            return f"{hours}:{minutes:02}:{seconds:02}"
-        else:
-            return f"{minutes}:{seconds:02}"
-    
-    def _calculate_estimated_completion(self) -> str:
-        """
-        Calculate estimated completion time based on current progress.
-        
-        Returns:
-            Formatted time string
-        """
-        summary = self.get_summary()
-        return summary.get("estimated_completion_time", "N/A")
-
-
-# Example code for testing
-if __name__ == "__main__":
-    # Initialize the monitor
-    monitor = CrawlerMonitor(urls_total=100)
-    
-    # Start monitoring
-    monitor.start()
-    
-    try:
-        # Simulate some tasks
-        for i in range(20):
-            task_id = str(uuid.uuid4())
-            url = f"https://example.com/page{i}"
-            monitor.add_task(task_id, url)
-            
-            # Simulate 20% of tasks are already running
-            if i < 4:
-                monitor.update_task(
-                    task_id=task_id,
-                    status=CrawlStatus.IN_PROGRESS,
-                    start_time=time.time() - 30,  # Started 30 seconds ago
-                    memory_usage=10.5
-                )
-                
-            # Simulate 10% of tasks are completed
-            if i >= 4 and i < 6:
-                start_time = time.time() - 60
-                end_time = time.time() - 15
-                monitor.update_task(
-                    task_id=task_id,
-                    status=CrawlStatus.IN_PROGRESS,
-                    start_time=start_time,
-                    memory_usage=8.2
-                )
-                monitor.update_task(
-                    task_id=task_id,
-                    status=CrawlStatus.COMPLETED,
-                    end_time=end_time,
-                    memory_usage=0,
-                    peak_memory=15.7
-                )
-                
-            # Simulate 5% of tasks fail
-            if i >= 6 and i < 7:
-                start_time = time.time() - 45
-                end_time = time.time() - 20
-                monitor.update_task(
-                    task_id=task_id,
-                    status=CrawlStatus.IN_PROGRESS,
-                    start_time=start_time,
-                    memory_usage=12.3
-                )
-                monitor.update_task(
-                    task_id=task_id,
-                    status=CrawlStatus.FAILED,
-                    end_time=end_time,
-                    memory_usage=0,
-                    peak_memory=18.2,
-                    error_message="Connection timeout"
-                )
-        
-        # Simulate memory pressure
-        monitor.update_memory_status("PRESSURE")
-        
-        # Simulate queue statistics
-        monitor.update_queue_statistics(
-            total_queued=16,  # 20 - 4 (in progress)
-            highest_wait_time=120.5,
-            avg_wait_time=60.2
-        )
-        
-        # Keep the monitor running for a demonstration
-        print("Crawler Monitor is running. Press 'q' to exit.")
-        while monitor.is_running:
-            time.sleep(0.1)
-            
-    except KeyboardInterrupt:
-        print("\nExiting crawler monitor...")
-    finally:
-        # Stop the monitor
-        monitor.stop()
-        print("Crawler monitor exited successfully.")

crawl4ai/config.py

@@ -4,85 +4,31 @@ from dotenv import load_dotenv
 load_dotenv()  # Load environment variables from .env file
 
 # Default provider, ONLY used when the extraction strategy is LLMExtractionStrategy
-DEFAULT_PROVIDER = "openai/gpt-4o"
-DEFAULT_PROVIDER_API_KEY = "OPENAI_API_KEY"
+DEFAULT_PROVIDER = "openai/gpt-4-turbo"
 MODEL_REPO_BRANCH = "new-release-0.0.2"
 # Provider-model dictionary, ONLY used when the extraction strategy is LLMExtractionStrategy
 PROVIDER_MODELS = {
-    "ollama/llama3": "no-token-needed",  # Any model from Ollama no need for API token
+    "ollama/llama3": "no-token-needed", # Any model from Ollama no need for API token
     "groq/llama3-70b-8192": os.getenv("GROQ_API_KEY"),
     "groq/llama3-8b-8192": os.getenv("GROQ_API_KEY"),
-    "openai/gpt-4o-mini": os.getenv("OPENAI_API_KEY"),
+    "openai/gpt-3.5-turbo": os.getenv("OPENAI_API_KEY"),
+    "openai/gpt-4-turbo": os.getenv("OPENAI_API_KEY"),
     "openai/gpt-4o": os.getenv("OPENAI_API_KEY"),
-    "openai/o1-mini": os.getenv("OPENAI_API_KEY"),
-    "openai/o1-preview": os.getenv("OPENAI_API_KEY"),
-    "openai/o3-mini": os.getenv("OPENAI_API_KEY"),
-    "openai/o3-mini-high": os.getenv("OPENAI_API_KEY"),
     "anthropic/claude-3-haiku-20240307": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-opus-20240229": os.getenv("ANTHROPIC_API_KEY"),
     "anthropic/claude-3-sonnet-20240229": os.getenv("ANTHROPIC_API_KEY"),
-    "anthropic/claude-3-5-sonnet-20240620": os.getenv("ANTHROPIC_API_KEY"),
-    "gemini/gemini-pro": os.getenv("GEMINI_API_KEY"),
-    'gemini/gemini-1.5-pro': os.getenv("GEMINI_API_KEY"),
-    'gemini/gemini-2.0-flash': os.getenv("GEMINI_API_KEY"),
-    'gemini/gemini-2.0-flash-exp': os.getenv("GEMINI_API_KEY"),
-    'gemini/gemini-2.0-flash-lite-preview-02-05': os.getenv("GEMINI_API_KEY"),
-    "deepseek/deepseek-chat": os.getenv("DEEPSEEK_API_KEY"),
-}
-PROVIDER_MODELS_PREFIXES = {
-    "ollama": "no-token-needed",  # Any model from Ollama no need for API token
-    "groq": os.getenv("GROQ_API_KEY"),
-    "openai": os.getenv("OPENAI_API_KEY"),
-    "anthropic": os.getenv("ANTHROPIC_API_KEY"),
-    "gemini": os.getenv("GEMINI_API_KEY"),
-    "deepseek": os.getenv("DEEPSEEK_API_KEY"),
 }
 
+
 # Chunk token threshold
-CHUNK_TOKEN_THRESHOLD = 2**11  # 2048 tokens
+CHUNK_TOKEN_THRESHOLD = 500
 OVERLAP_RATE = 0.1
 WORD_TOKEN_RATE = 1.3
 
-# Threshold for the minimum number of word in a HTML tag to be considered
+# Threshold for the minimum number of word in a HTML tag to be considered 
 MIN_WORD_THRESHOLD = 1
 IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD = 1
 
-IMPORTANT_ATTRS = ["src", "href", "alt", "title", "width", "height"]
-ONLY_TEXT_ELIGIBLE_TAGS = [
-    "b",
-    "i",
-    "u",
-    "span",
-    "del",
-    "ins",
-    "sub",
-    "sup",
-    "strong",
-    "em",
-    "code",
-    "kbd",
-    "var",
-    "s",
-    "q",
-    "abbr",
-    "cite",
-    "dfn",
-    "time",
-    "small",
-    "mark",
-]
-SOCIAL_MEDIA_DOMAINS = [
-    "facebook.com",
-    "twitter.com",
-    "x.com",
-    "linkedin.com",
-    "instagram.com",
-    "pinterest.com",
-    "tiktok.com",
-    "snapchat.com",
-    "reddit.com",
-]
-
 # Threshold for the Image extraction - Range is 1 to 6
 # Images are scored based on point based system, to filter based on usefulness. Points are assigned
 # to each image based on the following aspects.
@@ -92,55 +38,3 @@ SOCIAL_MEDIA_DOMAINS = [
 # If image format is in jpg, png or webp
 # If image is in the first half of the total images extracted from the page
 IMAGE_SCORE_THRESHOLD = 2
-
-MAX_METRICS_HISTORY = 1000
-
-NEED_MIGRATION = True
-URL_LOG_SHORTEN_LENGTH = 30
-SHOW_DEPRECATION_WARNINGS = True
-SCREENSHOT_HEIGHT_TRESHOLD = 10000
-PAGE_TIMEOUT = 60000
-DOWNLOAD_PAGE_TIMEOUT = 60000
-
-# Global user settings with descriptions and default values
-USER_SETTINGS = {
-    "DEFAULT_LLM_PROVIDER": {
-        "default": "openai/gpt-4o",
-        "description": "Default LLM provider in 'company/model' format (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')",
-        "type": "string"
-    },
-    "DEFAULT_LLM_PROVIDER_TOKEN": {
-        "default": "",
-        "description": "API token for the default LLM provider",
-        "type": "string",
-        "secret": True
-    },
-    "VERBOSE": {
-        "default": False,
-        "description": "Enable verbose output for all commands",
-        "type": "boolean"
-    },
-    "BROWSER_HEADLESS": {
-        "default": True,
-        "description": "Run browser in headless mode by default",
-        "type": "boolean"
-    },
-    "BROWSER_TYPE": {
-        "default": "chromium",
-        "description": "Default browser type (chromium or firefox)",
-        "type": "string",
-        "options": ["chromium", "firefox"]
-    },
-    "CACHE_MODE": {
-        "default": "bypass",
-        "description": "Default cache mode (bypass, use, or refresh)",
-        "type": "string",
-        "options": ["bypass", "use", "refresh"]
-    },
-    "USER_AGENT_MODE": {
-        "default": "default",
-        "description": "Default user agent mode (default, random, or mobile)",
-        "type": "string",
-        "options": ["default", "random", "mobile"]
-    }
-}

crawl4ai/content_scraping_strategy.py

@@ -1,900 +0,0 @@
-import re
-from itertools import chain
-from abc import ABC, abstractmethod
-from typing import Dict, Any, Optional
-from bs4 import BeautifulSoup
-import asyncio
-import requests
-from .config import (
-    MIN_WORD_THRESHOLD,
-    IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD,
-    IMAGE_SCORE_THRESHOLD,
-    ONLY_TEXT_ELIGIBLE_TAGS,
-    IMPORTANT_ATTRS,
-    SOCIAL_MEDIA_DOMAINS,
-)
-from bs4 import NavigableString, Comment
-from bs4 import PageElement, Tag
-from urllib.parse import urljoin
-from requests.exceptions import InvalidSchema
-from .utils import (
-    extract_metadata,
-    normalize_url,
-    is_external_url,
-    get_base_domain,
-    extract_metadata_using_lxml,
-    extract_page_context,
-    calculate_link_intrinsic_score,
-)
-from lxml import etree
-from lxml import html as lhtml
-from typing import List
-from .models import ScrapingResult, MediaItem, Link, Media, Links
-import copy
-
-# Pre-compile regular expressions for Open Graph and Twitter metadata
-OG_REGEX = re.compile(r"^og:")
-TWITTER_REGEX = re.compile(r"^twitter:")
-DIMENSION_REGEX = re.compile(r"(\d+)(\D*)")
-
-
-# Function to parse srcset
-def parse_srcset(s: str) -> List[Dict]:
-    if not s:
-        return []
-    variants = []
-    for part in s.split(","):
-        part = part.strip()
-        if not part:
-            continue
-        parts = part.split()
-        if len(parts) >= 1:
-            url = parts[0]
-            width = (
-                parts[1].rstrip("w").split('.')[0]
-                if len(parts) > 1 and parts[1].endswith("w")
-                else None
-            )
-            variants.append({"url": url, "width": width})
-    return variants
-
-
-# Function to parse image height/width value and units
-def parse_dimension(dimension):
-    if dimension:
-        # match = re.match(r"(\d+)(\D*)", dimension)
-        match = DIMENSION_REGEX.match(dimension)
-        if match:
-            number = int(match.group(1))
-            unit = match.group(2) or "px"  # Default unit is 'px' if not specified
-            return number, unit
-    return None, None
-
-
-# Fetch image file metadata to extract size and extension
-def fetch_image_file_size(img, base_url):
-    # If src is relative path construct full URL, if not it may be CDN URL
-    img_url = urljoin(base_url, img.get("src"))
-    try:
-        response = requests.head(img_url)
-        if response.status_code == 200:
-            return response.headers.get("Content-Length", None)
-        else:
-            print(f"Failed to retrieve file size for {img_url}")
-            return None
-    except InvalidSchema:
-        return None
-    finally:
-        return
-
-
-class ContentScrapingStrategy(ABC):
-    @abstractmethod
-    def scrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
-        pass
-
-    @abstractmethod
-    async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
-        pass
-
-
-class LXMLWebScrapingStrategy(ContentScrapingStrategy):
-    """
-    LXML-based implementation for fast web content scraping.
-    
-    This is the primary scraping strategy in Crawl4AI, providing high-performance
-    HTML parsing and content extraction using the lxml library.
-    
-    Note: WebScrapingStrategy is now an alias for this class to maintain
-    backward compatibility.
-    """
-    def __init__(self, logger=None):
-        self.logger = logger
-        self.DIMENSION_REGEX = re.compile(r"(\d+)(\D*)")
-        self.BASE64_PATTERN = re.compile(r'data:image/[^;]+;base64,([^"]+)')
-
-    def _log(self, level, message, tag="SCRAPE", **kwargs):
-        """Helper method to safely use logger."""
-        if self.logger:
-            log_method = getattr(self.logger, level)
-            log_method(message=message, tag=tag, **kwargs)
-
-    def scrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
-        """
-        Main entry point for content scraping.
-
-        Args:
-            url (str): The URL of the page to scrape.
-            html (str): The HTML content of the page.
-            **kwargs: Additional keyword arguments.
-
-        Returns:
-            ScrapingResult: A structured result containing the scraped content.
-        """
-        actual_url = kwargs.get("redirected_url", url)
-        raw_result = self._scrap(actual_url, html, **kwargs)
-        if raw_result is None:
-            return ScrapingResult(
-                cleaned_html="",
-                success=False,
-                media=Media(),
-                links=Links(),
-                metadata={},
-            )
-
-        # Convert media items
-        media = Media(
-            images=[
-                MediaItem(**img)
-                for img in raw_result.get("media", {}).get("images", [])
-                if img
-            ],
-            videos=[
-                MediaItem(**vid)
-                for vid in raw_result.get("media", {}).get("videos", [])
-                if vid
-            ],
-            audios=[
-                MediaItem(**aud)
-                for aud in raw_result.get("media", {}).get("audios", [])
-                if aud
-            ],
-            tables=raw_result.get("media", {}).get("tables", [])
-        )
-
-        # Convert links
-        links = Links(
-            internal=[
-                Link(**link)
-                for link in raw_result.get("links", {}).get("internal", [])
-                if link
-            ],
-            external=[
-                Link(**link)
-                for link in raw_result.get("links", {}).get("external", [])
-                if link
-            ],
-        )
-
-        return ScrapingResult(
-            cleaned_html=raw_result.get("cleaned_html", ""),
-            success=raw_result.get("success", False),
-            media=media,
-            links=links,
-            metadata=raw_result.get("metadata", {}),
-        )
-
-    async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
-        """
-        Main entry point for asynchronous content scraping.
-
-        Args:
-            url (str): The URL of the page to scrape.
-            html (str): The HTML content of the page.
-            **kwargs: Additional keyword arguments.
-
-        Returns:
-            ScrapingResult: A structured result containing the scraped content.
-        """
-        return await asyncio.to_thread(self.scrap, url, html, **kwargs)
-
-    def process_element(self, url, element: lhtml.HtmlElement, **kwargs) -> Dict[str, Any]:
-        """
-        Process an HTML element.
-
-        How it works:
-        1. Check if the element is an image, video, or audio.
-        2. Extract the element's attributes and content.
-        3. Process the element based on its type.
-        4. Return the processed element information.
-
-        Args:
-            url (str): The URL of the page containing the element.
-            element (lhtml.HtmlElement): The HTML element to process.
-            **kwargs: Additional keyword arguments.
-
-        Returns:
-            dict: A dictionary containing the processed element information.
-        """
-        media = {"images": [], "videos": [], "audios": [], "tables": []}
-        internal_links_dict = {}
-        external_links_dict = {}
-        self._process_element(
-            url, element, media, internal_links_dict, external_links_dict, **kwargs
-        )
-        return {
-            "media": media,
-            "internal_links_dict": internal_links_dict,
-            "external_links_dict": external_links_dict,
-        }
-
-    def _process_element(
-        self,
-        url: str,
-        element: lhtml.HtmlElement,
-        media: Dict[str, List],
-        internal_links_dict: Dict[str, Any],
-        external_links_dict: Dict[str, Any],
-        page_context: dict = None,
-        **kwargs,
-    ) -> bool:
-        base_domain = kwargs.get("base_domain", get_base_domain(url))
-        exclude_domains = set(kwargs.get("exclude_domains", []))
-
-        # Process links
-        try:
-            base_element = element.xpath("//head/base[@href]")
-            if base_element:
-                base_href = base_element[0].get("href", "").strip()
-                if base_href:
-                    url = base_href
-        except Exception as e:
-            self._log("error", f"Error extracting base URL: {str(e)}", "SCRAPE")
-            pass
-
-        for link in element.xpath(".//a[@href]"):
-            href = link.get("href", "").strip()
-            if not href:
-                continue
-
-            try:
-                normalized_href = normalize_url(
-                    href, url,
-                    preserve_https=kwargs.get('preserve_https_for_internal_links', False),
-                    original_scheme=kwargs.get('original_scheme')
-                )
-                link_data = {
-                    "href": normalized_href,
-                    "text": link.text_content().strip(),
-                    "title": link.get("title", "").strip(),
-                    "base_domain": base_domain,
-                }
-                
-                # Add intrinsic scoring if enabled
-                if kwargs.get("score_links", False) and page_context is not None:
-                    try:
-                        intrinsic_score = calculate_link_intrinsic_score(
-                            link_text=link_data["text"],
-                            url=normalized_href,
-                            title_attr=link_data["title"],
-                            class_attr=link.get("class", ""),
-                            rel_attr=link.get("rel", ""),
-                            page_context=page_context
-                        )
-                        link_data["intrinsic_score"] = intrinsic_score
-                    except Exception:
-                        # Fail gracefully - assign default score
-                        link_data["intrinsic_score"] = 0
-                else:
-                    # No scoring enabled - assign infinity (all links equal priority)
-                    link_data["intrinsic_score"] = 0
-
-                is_external = is_external_url(normalized_href, base_domain)
-                if is_external:
-                    link_base_domain = get_base_domain(normalized_href)
-                    link_data["base_domain"] = link_base_domain
-                    if (
-                        kwargs.get("exclude_external_links", False)
-                        or link_base_domain in exclude_domains
-                    ):
-                        link.getparent().remove(link)
-                        continue
-
-                    if normalized_href not in external_links_dict:
-                        external_links_dict[normalized_href] = link_data
-                else:
-                    if normalized_href not in internal_links_dict:
-                        internal_links_dict[normalized_href] = link_data
-
-            except Exception as e:
-                self._log("error", f"Error processing link: {str(e)}", "SCRAPE")
-                continue
-
-        # Process images
-        images = element.xpath(".//img")
-        total_images = len(images)
-
-        for idx, img in enumerate(images):
-            src = img.get("src") or ""
-            img_domain = get_base_domain(src)
-
-            # Decide if we need to exclude this image
-            # 1) If its domain is in exclude_domains, remove.
-            # 2) Or if exclude_external_images=True and it's an external domain, remove.
-            if (img_domain in exclude_domains) or (
-                kwargs.get("exclude_external_images", False)
-                and is_external_url(src, base_domain)
-            ):
-                parent = img.getparent()
-                if parent is not None:
-                    parent.remove(img)
-                continue
-
-            # Otherwise, process the image as usual.
-            try:
-                processed_images = self.process_image(
-                    img, url, idx, total_images, **kwargs
-                )
-                if processed_images:
-                    media["images"].extend(processed_images)
-            except Exception as e:
-                self._log("error", f"Error processing image: {str(e)}", "SCRAPE")
-
-        # Process videos and audios
-        for media_type in ["video", "audio"]:
-            for elem in element.xpath(f".//{media_type}"):
-                media_info = {
-                    "src": elem.get("src"),
-                    "alt": elem.get("alt"),
-                    "type": media_type,
-                    "description": self.find_closest_parent_with_useful_text(
-                        elem, **kwargs
-                    ),
-                }
-                media[f"{media_type}s"].append(media_info)
-
-                # Process source tags within media elements
-                for source in elem.xpath(".//source"):
-                    if src := source.get("src"):
-                        media[f"{media_type}s"].append({**media_info, "src": src})
-
-        # Clean up unwanted elements
-        if kwargs.get("remove_forms", False):
-            for form in element.xpath(".//form"):
-                form.getparent().remove(form)
-
-        if excluded_tags := kwargs.get("excluded_tags", []):
-            for tag in excluded_tags:
-                for elem in element.xpath(f".//{tag}"):
-                    elem.getparent().remove(elem)
-
-        if excluded_selector := kwargs.get("excluded_selector", ""):
-            try:
-                for elem in element.cssselect(excluded_selector):
-                    elem.getparent().remove(elem)
-            except Exception:
-                pass  # Invalid selector
-
-        return True
-
-    def find_closest_parent_with_useful_text(
-        self, element: lhtml.HtmlElement, **kwargs
-    ) -> Optional[str]:
-        image_description_min_word_threshold = kwargs.get(
-            "image_description_min_word_threshold", IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD
-        )
-        current = element
-        while current is not None:
-            if (
-                current.text
-                and len(current.text_content().split())
-                >= image_description_min_word_threshold
-            ):
-                return current.text_content().strip()
-            current = current.getparent()
-        return None
-
-    def flatten_nested_elements(self, element: lhtml.HtmlElement) -> lhtml.HtmlElement:
-        """Flatten nested elements of the same type in LXML tree"""
-        if len(element) == 1 and element.tag == element[0].tag:
-            return self.flatten_nested_elements(element[0])
-
-        for child in element:
-            child_idx = element.index(child)
-            flattened_child = self.flatten_nested_elements(child)
-            if flattened_child is not child:  # Only replace if actually flattened
-                element[child_idx] = flattened_child
-
-        return element
-
-    def process_image(
-        self, img: lhtml.HtmlElement, url: str, index: int, total_images: int, **kwargs
-    ) -> Optional[List[Dict]]:
-        # Quick validation checks
-        style = img.get("style", "")
-        alt = img.get("alt", "")
-        src = img.get("src", "")
-        data_src = img.get("data-src", "")
-        srcset = img.get("srcset", "")
-        data_srcset = img.get("data-srcset", "")
-
-        if "display:none" in style:
-            return None
-
-        parent = img.getparent()
-        if parent.tag in ["button", "input"]:
-            return None
-
-        parent_classes = parent.get("class", "").split()
-        if any(
-            "button" in cls or "icon" in cls or "logo" in cls for cls in parent_classes
-        ):
-            return None
-
-        # If src is in class or alt, likely an icon
-        if (src and any(c in src for c in ["button", "icon", "logo"])) or (
-            alt and any(c in alt for c in ["button", "icon", "logo"])
-        ):
-            return None
-
-        # Score calculation
-        score = 0
-        if (width := img.get("width")) and width.isdigit():
-            score += 1 if int(width) > 150 else 0
-        if (height := img.get("height")) and height.isdigit():
-            score += 1 if int(height) > 150 else 0
-        if alt:
-            score += 1
-        score += index / total_images < 0.5
-
-        # Check formats in all possible sources
-        image_formats = {"jpg", "jpeg", "png", "webp", "avif", "gif"}
-        detected_format = None
-        for url in [src, data_src, srcset, data_srcset]:
-            if url:
-                format_matches = [fmt for fmt in image_formats if fmt in url.lower()]
-                if format_matches:
-                    detected_format = format_matches[0]
-                    score += 1
-                    break
-
-        if srcset or data_srcset:
-            score += 1
-
-        if picture := img.xpath("./ancestor::picture[1]"):
-            score += 1
-
-        if score <= kwargs.get("image_score_threshold", IMAGE_SCORE_THRESHOLD):
-            return None
-
-        # Process image variants
-        unique_urls = set()
-        image_variants = []
-        base_info = {
-            "alt": alt,
-            "desc": self.find_closest_parent_with_useful_text(img, **kwargs),
-            "score": score,
-            "type": "image",
-            "group_id": index,
-            "format": detected_format,
-        }
-
-        def add_variant(src: str, width: Optional[str] = None):
-            if src and not src.startswith("data:") and src not in unique_urls:
-                unique_urls.add(src)
-                variant = {**base_info, "src": src}
-                if width:
-                    variant["width"] = width
-                image_variants.append(variant)
-
-        # Add variants from different sources
-        add_variant(src)
-        add_variant(data_src)
-
-        for srcset_attr in [srcset, data_srcset]:
-            if srcset_attr:
-                for source in parse_srcset(srcset_attr):
-                    add_variant(source["url"], source["width"])
-
-        # Handle picture element
-        if picture:
-            for source in picture[0].xpath(".//source[@srcset]"):
-                if source_srcset := source.get("srcset"):
-                    for src_data in parse_srcset(source_srcset):
-                        add_variant(src_data["url"], src_data["width"])
-
-        # Check framework-specific attributes
-        for attr, value in img.attrib.items():
-            if (
-                attr.startswith("data-")
-                and ("src" in attr or "srcset" in attr)
-                and "http" in value
-            ):
-                add_variant(value)
-
-        return image_variants if image_variants else None
-
-    def remove_empty_elements_fast(self, root, word_count_threshold=5):
-        """
-        Remove elements that fall below the desired word threshold in a single pass from the bottom up.
-        Skips non-element nodes like HtmlComment and bypasses certain tags that are allowed to have no content.
-        """
-        bypass_tags = {
-            "a",
-            "img",
-            "br",
-            "hr",
-            "input",
-            "meta",
-            "link",
-            "source",
-            "track",
-            "wbr",
-            "tr",
-            "td",
-            "th",
-        }
-
-        for el in reversed(list(root.iterdescendants())):
-            if not isinstance(el, lhtml.HtmlElement):
-                continue
-
-            if el.tag in bypass_tags:
-                continue
-
-            text_content = (el.text_content() or "").strip()
-            if (
-                len(text_content.split()) < word_count_threshold
-                and not el.getchildren()
-            ):
-                parent = el.getparent()
-                if parent is not None:
-                    parent.remove(el)
-
-        return root
-
-    def remove_unwanted_attributes_fast(
-        self, root: lhtml.HtmlElement, important_attrs=None, keep_data_attributes=False
-    ) -> lhtml.HtmlElement:
-        """
-        Removes all attributes from each element (including root) except those in `important_attrs`.
-        If `keep_data_attributes=True`, also retain any attribute starting with 'data-'.
-
-        Returns the same root element, mutated in-place, for fluent usage.
-        """
-        if important_attrs is None:
-            important_attrs = set(IMPORTANT_ATTRS)
-
-        # If you want to handle the root as well, use 'include_self=True'
-        # so you don't miss attributes on the top-level element.
-        # Manually include the root, then all its descendants
-        for el in chain((root,), root.iterdescendants()):
-            # We only remove attributes on HtmlElement nodes, skip comments or text nodes
-            if not isinstance(el, lhtml.HtmlElement):
-                continue
-
-            old_attribs = dict(el.attrib)
-            new_attribs = {}
-
-            for attr_name, attr_val in old_attribs.items():
-                # If it's an important attribute, keep it
-                if attr_name in important_attrs:
-                    new_attribs[attr_name] = attr_val
-                # Or if keep_data_attributes is True and it's a 'data-*' attribute
-                elif keep_data_attributes and attr_name.startswith("data-"):
-                    new_attribs[attr_name] = attr_val
-
-            # Clear old attributes and set the filtered set
-            el.attrib.clear()
-            el.attrib.update(new_attribs)
-
-        return root
-
-
-    def _scrap(
-        self,
-        url: str,
-        html: str,
-        word_count_threshold: int = MIN_WORD_THRESHOLD,
-        css_selector: str = None,
-        target_elements: List[str] = None,
-        **kwargs,
-    ) -> Dict[str, Any]:
-        if not html:
-            return None
-
-        success = True
-        try:
-            doc = lhtml.document_fromstring(html)
-            # Match BeautifulSoup's behavior of using body or full doc
-            # body = doc.xpath('//body')[0] if doc.xpath('//body') else doc
-            body = doc
-
-            base_domain = get_base_domain(url)
-            
-            # Extract page context for link scoring (if enabled) - do this BEFORE any removals
-            page_context = None
-            if kwargs.get("score_links", False):
-                try:
-                    # Extract title
-                    title_elements = doc.xpath('//title')
-                    page_title = title_elements[0].text_content() if title_elements else ""
-                    
-                    # Extract headlines
-                    headlines = []
-                    for tag in ['h1', 'h2', 'h3']:
-                        elements = doc.xpath(f'//{tag}')
-                        for el in elements:
-                            text = el.text_content().strip()
-                            if text:
-                                headlines.append(text)
-                    headlines_text = ' '.join(headlines)
-                    
-                    # Extract meta description
-                    meta_desc_elements = doc.xpath('//meta[@name="description"]/@content')
-                    meta_description = meta_desc_elements[0] if meta_desc_elements else ""
-                    
-                    # Create page context
-                    page_context = extract_page_context(page_title, headlines_text, meta_description, url)
-                except Exception:
-                    page_context = {}  # Fail gracefully
-            
-            # Early removal of all images if exclude_all_images is set
-            # This is more efficient in lxml as we remove elements before any processing
-            if kwargs.get("exclude_all_images", False):
-                for img in body.xpath('//img'):
-                    if img.getparent() is not None:
-                        img.getparent().remove(img)
-
-            # Add comment removal
-            if kwargs.get("remove_comments", False):
-                comments = body.xpath("//comment()")
-                for comment in comments:
-                    comment.getparent().remove(comment)
-
-            # Handle tag-based removal first
-            excluded_tags = set(kwargs.get("excluded_tags", []) or [])
-            if excluded_tags:
-                for tag in excluded_tags:
-                    for element in body.xpath(f".//{tag}"):
-                        if element.getparent() is not None:
-                            element.getparent().remove(element)
-
-            # Handle CSS selector-based exclusion
-            excluded_selector = kwargs.get("excluded_selector", "")
-            if excluded_selector:
-                try:
-                    for element in body.cssselect(excluded_selector):
-                        if element.getparent() is not None:
-                            element.getparent().remove(element)
-                except Exception as e:
-                    self._log(
-                        "error", f"Error with excluded CSS selector: {str(e)}", "SCRAPE"
-                    )
-
-            # Extract metadata before any content filtering
-            try:
-                meta = extract_metadata_using_lxml(
-                    "", doc
-                )  # Using same function as BeautifulSoup version
-            except Exception as e:
-                self._log("error", f"Error extracting metadata: {str(e)}", "SCRAPE")
-                meta = {}
-
-            content_element = None
-            if target_elements:
-                try:
-                    for_content_targeted_element = []
-                    for target_element in target_elements:
-                        for_content_targeted_element.extend(body.cssselect(target_element))
-                    content_element = lhtml.Element("div")
-                    content_element.extend(copy.deepcopy(for_content_targeted_element))
-                except Exception as e:
-                    self._log("error", f"Error with target element detection: {str(e)}", "SCRAPE")
-                    return None
-            else:
-                content_element = body
-
-            # Remove script and style tags
-            for tag in ["script", "style", "link", "meta", "noscript"]:
-                for element in body.xpath(f".//{tag}"):
-                    if element.getparent() is not None:
-                        element.getparent().remove(element)
-
-            # Handle social media and domain exclusions
-            kwargs["exclude_domains"] = set(kwargs.get("exclude_domains", []))
-            if kwargs.get("exclude_social_media_links", False):
-                kwargs["exclude_social_media_domains"] = set(
-                    kwargs.get("exclude_social_media_domains", [])
-                    + SOCIAL_MEDIA_DOMAINS
-                )
-                kwargs["exclude_domains"].update(kwargs["exclude_social_media_domains"])
-
-            # Process forms if needed
-            if kwargs.get("remove_forms", False):
-                for form in body.xpath(".//form"):
-                    if form.getparent() is not None:
-                        form.getparent().remove(form)
-
-            # Process content
-            media = {"images": [], "videos": [], "audios": [], "tables": []}
-            internal_links_dict = {}
-            external_links_dict = {}
-
-            self._process_element(
-                url,
-                body,
-                media,
-                internal_links_dict,
-                external_links_dict,
-                page_context=page_context,
-                base_domain=base_domain,
-                **kwargs,
-            )
-
-            # Extract tables using the table extraction strategy if provided
-            if 'table' not in excluded_tags:
-                table_extraction = kwargs.get('table_extraction')
-                if table_extraction:
-                    # Pass logger to the strategy if it doesn't have one
-                    if not table_extraction.logger:
-                        table_extraction.logger = self.logger
-                    # Extract tables using the strategy
-                    extracted_tables = table_extraction.extract_tables(body, **kwargs)
-                    media["tables"].extend(extracted_tables)
-
-            # Handle only_text option
-            if kwargs.get("only_text", False):
-                for tag in ONLY_TEXT_ELIGIBLE_TAGS:
-                    for element in body.xpath(f".//{tag}"):
-                        if element.text:
-                            new_text = lhtml.Element("span")
-                            new_text.text = element.text_content()
-                            if element.getparent() is not None:
-                                element.getparent().replace(element, new_text)
-
-            # Clean base64 images
-            for img in body.xpath(".//img[@src]"):
-                src = img.get("src", "")
-                if self.BASE64_PATTERN.match(src):
-                    img.set("src", self.BASE64_PATTERN.sub("", src))
-
-            # Remove empty elements
-            self.remove_empty_elements_fast(body, 1)
-
-            # Remove unneeded attributes
-            self.remove_unwanted_attributes_fast(
-                body, keep_data_attributes=kwargs.get("keep_data_attributes", False)
-            )
-
-            # Generate output HTML
-            cleaned_html = lhtml.tostring(
-                # body,   
-                content_element,
-                encoding="unicode",
-                pretty_print=True,
-                method="html",
-                with_tail=False,
-            ).strip()
-            
-            # Create links dictionary in the format expected by LinkPreview
-            links = {
-                "internal": list(internal_links_dict.values()),
-                "external": list(external_links_dict.values()),
-            }
-            
-            # Extract head content for links if configured
-            link_preview_config = kwargs.get("link_preview_config")
-            if link_preview_config is not None:
-                try:
-                    import asyncio
-                    from .link_preview import LinkPreview
-                    from .models import Links, Link
-                    
-                    verbose = link_preview_config.verbose
-                    
-                    if verbose:
-                        self._log("info", "Starting link head extraction for {internal} internal and {external} external links",
-                                  params={"internal": len(links["internal"]), "external": len(links["external"])}, tag="LINK_EXTRACT")
-                    
-                    # Convert dict links to Link objects
-                    internal_links = [Link(**link_data) for link_data in links["internal"]]
-                    external_links = [Link(**link_data) for link_data in links["external"]]
-                    links_obj = Links(internal=internal_links, external=external_links)
-                    
-                    # Create a config object for LinkPreview
-                    class TempCrawlerRunConfig:
-                        def __init__(self, link_config, score_links):
-                            self.link_preview_config = link_config
-                            self.score_links = score_links
-                    
-                    config = TempCrawlerRunConfig(link_preview_config, kwargs.get("score_links", False))
-                    
-                    # Extract head content (run async operation in sync context)
-                    async def extract_links():
-                        async with LinkPreview(self.logger) as extractor:
-                            return await extractor.extract_link_heads(links_obj, config)
-                    
-                    # Run the async operation
-                    try:
-                        # Check if we're already in an async context
-                        loop = asyncio.get_running_loop()
-                        # If we're in an async context, we need to run in a thread
-                        import concurrent.futures
-                        with concurrent.futures.ThreadPoolExecutor() as executor:
-                            future = executor.submit(asyncio.run, extract_links())
-                            updated_links = future.result()
-                    except RuntimeError:
-                        # No running loop, we can use asyncio.run directly
-                        updated_links = asyncio.run(extract_links())
-                    
-                    # Convert back to dict format
-                    links["internal"] = [link.dict() for link in updated_links.internal]
-                    links["external"] = [link.dict() for link in updated_links.external]
-                    
-                    if verbose:
-                        successful_internal = len([l for l in updated_links.internal if l.head_extraction_status == "valid"])
-                        successful_external = len([l for l in updated_links.external if l.head_extraction_status == "valid"])
-                        self._log("info", "Link head extraction completed: {internal_success}/{internal_total} internal, {external_success}/{external_total} external",
-                                  params={
-                                      "internal_success": successful_internal,
-                                      "internal_total": len(updated_links.internal),
-                                      "external_success": successful_external,
-                                      "external_total": len(updated_links.external)
-                                  }, tag="LINK_EXTRACT")
-                    else:
-                        self._log("info", "Link head extraction completed successfully", tag="LINK_EXTRACT")
-                        
-                except Exception as e:
-                    self._log("error", f"Error during link head extraction: {str(e)}", tag="LINK_EXTRACT")
-                    # Continue with original links if head extraction fails
-            
-            return {
-                "cleaned_html": cleaned_html,
-                "success": success,
-                "media": media,
-                "links": links,
-                "metadata": meta,
-            }
-
-        except Exception as e:
-            self._log("error", f"Error processing HTML: {str(e)}", "SCRAPE")
-            # Create error message in case of failure
-            error_body = lhtml.Element("div")
-            # Use etree.SubElement rather than lhtml.SubElement
-            error_div = etree.SubElement(error_body, "div", id="crawl4ai_error_message")
-            error_div.text = f"""
-            Crawl4AI Error: This page is not fully supported.
-            
-            Error Message: {str(e)}
-            
-            Possible reasons:
-            1. The page may have restrictions that prevent crawling.
-            2. The page might not be fully loaded.
-            
-            Suggestions:
-            - Try calling the crawl function with these parameters:
-            magic=True,
-            - Set headless=False to visualize what's happening on the page.
-            
-            If the issue persists, please check the page's structure and any potential anti-crawling measures.
-            """
-            cleaned_html = lhtml.tostring(
-                error_body, encoding="unicode", pretty_print=True
-            )
-            return {
-                "cleaned_html": cleaned_html,
-                "success": False,
-                "media": {
-                    "images": [],
-                    "videos": [],
-                    "audios": [],
-                    "tables": []
-                },
-                "links": {"internal": [], "external": []},
-                "metadata": {},
-            }
-
-
-# Backward compatibility alias
-WebScrapingStrategy = LXMLWebScrapingStrategy

crawl4ai/content_scrapping_strategy.py

@@ -0,0 +1,301 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+from bs4 import BeautifulSoup
+from concurrent.futures import ThreadPoolExecutor
+import asyncio, requests, re, os
+from .config import *
+from bs4 import element, NavigableString, Comment
+from urllib.parse import urljoin
+from requests.exceptions import InvalidSchema
+
+from .utils import (
+    sanitize_input_encode,
+    sanitize_html,
+    extract_metadata,
+    InvalidCSSSelectorError,
+    CustomHTML2Text
+)
+
+class ContentScrappingStrategy(ABC):
+    @abstractmethod
+    def scrap(self, url: str, html: str, **kwargs) -> Dict[str, Any]:
+        pass
+
+    @abstractmethod
+    async def ascrap(self, url: str, html: str, **kwargs) -> Dict[str, Any]:
+        pass
+
+class WebScrappingStrategy(ContentScrappingStrategy):
+    def scrap(self, url: str, html: str, **kwargs) -> Dict[str, Any]:
+        return self._get_content_of_website_optimized(url, html, is_async=False, **kwargs)
+
+    async def ascrap(self, url: str, html: str, **kwargs) -> Dict[str, Any]:
+        return await asyncio.to_thread(self._get_content_of_website_optimized, url, html, **kwargs)
+
+    def _get_content_of_website_optimized(self, url: str, html: str, word_count_threshold: int = MIN_WORD_THRESHOLD, css_selector: str = None, **kwargs) -> Dict[str, Any]:
+        if not html:
+            return None
+
+        soup = BeautifulSoup(html, 'html.parser')
+        body = soup.body
+        
+        image_description_min_word_threshold = kwargs.get('image_description_min_word_threshold', IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD)
+
+        for tag in kwargs.get('excluded_tags', []) or []:
+            for el in body.select(tag):
+                el.decompose()
+        
+        if css_selector:
+            selected_elements = body.select(css_selector)
+            if not selected_elements:
+                return {
+                    'markdown': '',
+                    'cleaned_html': '',
+                    'success': True,
+                    'media': {'images': [], 'videos': [], 'audios': []},
+                    'links': {'internal': [], 'external': []},
+                    'metadata': {},
+                    'message': f"No elements found for CSS selector: {css_selector}"
+                }
+                # raise InvalidCSSSelectorError(f"Invalid CSS selector, No elements found for CSS selector: {css_selector}")
+            body = soup.new_tag('div')
+            for el in selected_elements:
+                body.append(el)
+
+        links = {'internal': [], 'external': []}
+        media = {'images': [], 'videos': [], 'audios': []}
+
+        # Extract meaningful text for media files from closest parent
+        def find_closest_parent_with_useful_text(tag):
+                current_tag = tag
+                while current_tag:
+                    current_tag = current_tag.parent
+                    # Get the text content of the parent tag
+                    if current_tag:
+                        text_content = current_tag.get_text(separator=' ',strip=True)
+                        # Check if the text content has at least word_count_threshold
+                        if len(text_content.split()) >= image_description_min_word_threshold:
+                            return text_content
+                return None
+
+        def process_image(img, url, index, total_images):
+            #Check if an image has valid display and inside undesired html elements
+            def is_valid_image(img, parent, parent_classes):
+                style = img.get('style', '')
+                src = img.get('src', '')
+                classes_to_check = ['button', 'icon', 'logo']
+                tags_to_check = ['button', 'input']
+                return all([
+                    'display:none' not in style,
+                    src,
+                    not any(s in var for var in [src, img.get('alt', ''), *parent_classes] for s in classes_to_check),
+                    parent.name not in tags_to_check
+                ])
+
+            #Score an image for it's usefulness
+            def score_image_for_usefulness(img, base_url, index, images_count):
+                # Function to parse image height/width value and units
+                def parse_dimension(dimension):
+                    if dimension:
+                        match = re.match(r"(\d+)(\D*)", dimension)
+                        if match:
+                            number = int(match.group(1))
+                            unit = match.group(2) or 'px'  # Default unit is 'px' if not specified
+                            return number, unit
+                    return None, None
+
+                # Fetch image file metadata to extract size and extension
+                def fetch_image_file_size(img, base_url):
+                    #If src is relative path construct full URL, if not it may be CDN URL
+                    img_url = urljoin(base_url,img.get('src'))
+                    try:
+                        response = requests.head(img_url)
+                        if response.status_code == 200:
+                            return response.headers.get('Content-Length',None)
+                        else:
+                            print(f"Failed to retrieve file size for {img_url}")
+                            return None
+                    except InvalidSchema as e:
+                        return None
+                    finally:
+                        return
+
+                image_height = img.get('height')
+                height_value, height_unit = parse_dimension(image_height)
+                image_width =  img.get('width')
+                width_value, width_unit = parse_dimension(image_width)
+                image_size = 0 #int(fetch_image_file_size(img,base_url) or 0)
+                image_format = os.path.splitext(img.get('src',''))[1].lower()
+                # Remove . from format
+                image_format = image_format.strip('.').split('?')[0]
+                score = 0
+                if height_value:
+                    if height_unit == 'px' and height_value > 150:
+                        score += 1
+                    if height_unit in ['%','vh','vmin','vmax'] and height_value >30:
+                        score += 1
+                if width_value:
+                    if width_unit == 'px' and width_value > 150:
+                        score += 1
+                    if width_unit in ['%','vh','vmin','vmax'] and width_value >30:
+                        score += 1
+                if image_size > 10000:
+                    score += 1
+                if img.get('alt') != '':
+                    score+=1
+                if any(image_format==format for format in ['jpg','png','webp']):
+                    score+=1
+                if index/images_count<0.5:
+                    score+=1
+                return score
+
+            if not is_valid_image(img, img.parent, img.parent.get('class', [])):
+                return None
+            score = score_image_for_usefulness(img, url, index, total_images)
+            if score <= IMAGE_SCORE_THRESHOLD:
+                return None
+            return {
+                'src': img.get('src', ''),
+                'data-src': img.get('data-src', ''),
+                'alt': img.get('alt', ''),
+                'desc': find_closest_parent_with_useful_text(img),
+                'score': score,
+                'type': 'image'
+            }
+
+        def process_element(element: element.PageElement) -> bool:
+            try:
+                if isinstance(element, NavigableString):
+                    if isinstance(element, Comment):
+                        element.extract()
+                    return False
+                
+                # if element.name == 'img':
+                #     process_image(element, url, 0, 1)
+                #     return True
+
+                if element.name in ['script', 'style', 'link', 'meta', 'noscript']:
+                    element.decompose()
+                    return False
+
+                keep_element = False
+
+                if element.name == 'a' and element.get('href'):
+                    href = element['href']
+                    url_base = url.split('/')[2]
+                    link_data = {'href': href, 'text': element.get_text()}
+                    if href.startswith('http') and url_base not in href:
+                        links['external'].append(link_data)
+                    else:
+                        links['internal'].append(link_data)
+                    keep_element = True
+
+                elif element.name == 'img':
+                    return True  # Always keep image elements
+
+                elif element.name in ['video', 'audio']:
+                    media[f"{element.name}s"].append({
+                        'src': element.get('src'),
+                        'alt': element.get('alt'),
+                        'type': element.name,
+                        'description': find_closest_parent_with_useful_text(element)
+                    })
+                    source_tags = element.find_all('source')
+                    for source_tag in source_tags:
+                        media[f"{element.name}s"].append({
+                        'src': source_tag.get('src'),
+                        'alt': element.get('alt'),
+                        'type': element.name,
+                        'description': find_closest_parent_with_useful_text(element)
+                    })
+                    return True  # Always keep video and audio elements
+
+                if element.name != 'pre':
+                    if element.name in ['b', 'i', 'u', 'span', 'del', 'ins', 'sub', 'sup', 'strong', 'em', 'code', 'kbd', 'var', 's', 'q', 'abbr', 'cite', 'dfn', 'time', 'small', 'mark']:
+                        if kwargs.get('only_text', False):
+                            element.replace_with(element.get_text())
+                        else:
+                            element.unwrap()
+                    elif element.name != 'img':
+                        element.attrs = {}
+
+                # Process children
+                for child in list(element.children):
+                    if isinstance(child, NavigableString) and not isinstance(child, Comment):
+                        if len(child.strip()) > 0:
+                            keep_element = True
+                    else:
+                        if process_element(child):
+                            keep_element = True
+                    
+
+                # Check word count
+                if not keep_element:
+                    word_count = len(element.get_text(strip=True).split())
+                    keep_element = word_count >= word_count_threshold
+
+                if not keep_element:
+                    element.decompose()
+
+                return keep_element
+            except Exception as e:
+                print('Error processing element:', str(e))
+                return False
+
+        #process images by filtering and extracting contextual text from the page
+        # imgs = body.find_all('img')
+        # media['images'] = [
+        #     result for result in
+        #     (process_image(img, url, i, len(imgs)) for i, img in enumerate(imgs))
+        #     if result is not None
+        # ]
+        
+        process_element(body)
+
+        # # Process images using ThreadPoolExecutor
+        imgs = body.find_all('img')
+        with ThreadPoolExecutor() as executor:
+            image_results = list(executor.map(process_image, imgs, [url]*len(imgs), range(len(imgs)), [len(imgs)]*len(imgs)))
+        media['images'] = [result for result in image_results if result is not None]
+
+        def flatten_nested_elements(node):
+            if isinstance(node, NavigableString):
+                return node
+            if len(node.contents) == 1 and isinstance(node.contents[0], element.Tag) and node.contents[0].name == node.name:
+                return flatten_nested_elements(node.contents[0])
+            node.contents = [flatten_nested_elements(child) for child in node.contents]
+            return node
+
+        body = flatten_nested_elements(body)
+        base64_pattern = re.compile(r'data:image/[^;]+;base64,([^"]+)')
+        for img in imgs:
+            src = img.get('src', '')
+            if base64_pattern.match(src):
+                # Replace base64 data with empty string
+                img['src'] = base64_pattern.sub('', src)
+        cleaned_html = str(body).replace('\n\n', '\n').replace('  ', ' ')
+
+        h = CustomHTML2Text()
+        h.ignore_links = True
+        h.body_width = 0
+        try:
+            markdown = h.handle(cleaned_html)
+        except Exception as e:
+            markdown = h.handle(sanitize_html(cleaned_html))
+        markdown = markdown.replace('    ```', '```')
+
+        try:
+            meta = extract_metadata(html, soup)
+        except Exception as e:
+            print('Error extracting metadata:', str(e))
+            meta = {}
+
+        cleaned_html = sanitize_html(cleaned_html)
+        return {
+            'markdown': markdown,
+            'cleaned_html': cleaned_html,
+            'success': True,
+            'media': media,
+            'links': links,
+            'metadata': meta
+        }

crawl4ai/crawler_strategy.py

@@ -15,53 +15,54 @@ import logging, time
 import base64
 from PIL import Image, ImageDraw, ImageFont
 from io import BytesIO
-from typing import Callable
+from typing import List, Callable
 import requests
 import os
 from pathlib import Path
 from .utils import *
 
-logger = logging.getLogger("selenium.webdriver.remote.remote_connection")
+logger = logging.getLogger('selenium.webdriver.remote.remote_connection')
 logger.setLevel(logging.WARNING)
 
-logger_driver = logging.getLogger("selenium.webdriver.common.service")
+logger_driver = logging.getLogger('selenium.webdriver.common.service')
 logger_driver.setLevel(logging.WARNING)
 
-urllib3_logger = logging.getLogger("urllib3.connectionpool")
+urllib3_logger = logging.getLogger('urllib3.connectionpool')
 urllib3_logger.setLevel(logging.WARNING)
 
 # Disable http.client logging
-http_client_logger = logging.getLogger("http.client")
+http_client_logger = logging.getLogger('http.client')
 http_client_logger.setLevel(logging.WARNING)
 
 # Disable driver_finder and service logging
-driver_finder_logger = logging.getLogger("selenium.webdriver.common.driver_finder")
+driver_finder_logger = logging.getLogger('selenium.webdriver.common.driver_finder')
 driver_finder_logger.setLevel(logging.WARNING)
 
 
+
+
 class CrawlerStrategy(ABC):
     @abstractmethod
     def crawl(self, url: str, **kwargs) -> str:
         pass
-
+    
     @abstractmethod
     def take_screenshot(self, save_path: str):
         pass
-
+    
     @abstractmethod
     def update_user_agent(self, user_agent: str):
         pass
-
+    
     @abstractmethod
     def set_hook(self, hook_type: str, hook: Callable):
         pass
 
-
 class CloudCrawlerStrategy(CrawlerStrategy):
-    def __init__(self, use_cached_html=False):
+    def __init__(self, use_cached_html = False):
         super().__init__()
         self.use_cached_html = use_cached_html
-
+        
     def crawl(self, url: str) -> str:
         data = {
             "urls": [url],
@@ -75,7 +76,6 @@ class CloudCrawlerStrategy(CrawlerStrategy):
         html = response["results"][0]["html"]
         return sanitize_input_encode(html)
 
-
 class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
     def __init__(self, use_cached_html=False, js_code=None, **kwargs):
         super().__init__()
@@ -87,25 +87,20 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         if kwargs.get("user_agent"):
             self.options.add_argument("--user-agent=" + kwargs.get("user_agent"))
         else:
-            user_agent = kwargs.get(
-                "user_agent",
-                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-            )
+            user_agent = kwargs.get("user_agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
             self.options.add_argument(f"--user-agent={user_agent}")
-            self.options.add_argument(
-                "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
-            )
-
+            self.options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+                  
         self.options.headless = kwargs.get("headless", True)
         if self.options.headless:
             self.options.add_argument("--headless")
-
-        self.options.add_argument("--disable-gpu")
+        
+        self.options.add_argument("--disable-gpu")  
         self.options.add_argument("--window-size=1920,1080")
         self.options.add_argument("--no-sandbox")
         self.options.add_argument("--disable-dev-shm-usage")
-        self.options.add_argument("--disable-blink-features=AutomationControlled")
-
+        self.options.add_argument("--disable-blink-features=AutomationControlled")     
+        
         # self.options.add_argument("--disable-dev-shm-usage")
         self.options.add_argument("--disable-gpu")
         # self.options.add_argument("--disable-extensions")
@@ -125,45 +120,48 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         self.use_cached_html = use_cached_html
         self.js_code = js_code
         self.verbose = kwargs.get("verbose", False)
-
+        
         # Hooks
         self.hooks = {
-            "on_driver_created": None,
-            "on_user_agent_updated": None,
-            "before_get_url": None,
-            "after_get_url": None,
-            "before_return_html": None,
+            'on_driver_created': None,
+            'on_user_agent_updated': None,
+            'before_get_url': None,
+            'after_get_url': None,
+            'before_return_html': None
         }
 
         # chromedriver_autoinstaller.install()
         # import chromedriver_autoinstaller
-        # crawl4ai_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+        # crawl4ai_folder = os.path.join(Path.home(), ".crawl4ai")
         # driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=self.options)
         # chromedriver_path = chromedriver_autoinstaller.install()
         # chromedriver_path = chromedriver_autoinstaller.utils.download_chromedriver()
         # self.service = Service(chromedriver_autoinstaller.install())
-
+        
+        
         # chromedriver_path = ChromeDriverManager().install()
         # self.service = Service(chromedriver_path)
         # self.service.log_path = "NUL"
         # self.driver = webdriver.Chrome(service=self.service, options=self.options)
-
+        
         # Use selenium-manager (built into Selenium 4.10.0+)
         self.service = Service()
         self.driver = webdriver.Chrome(options=self.options)
-
-        self.driver = self.execute_hook("on_driver_created", self.driver)
-
+        
+        self.driver = self.execute_hook('on_driver_created', self.driver)
+        
         if kwargs.get("cookies"):
             for cookie in kwargs.get("cookies"):
                 self.driver.add_cookie(cookie)
+            
+        
 
     def set_hook(self, hook_type: str, hook: Callable):
         if hook_type in self.hooks:
             self.hooks[hook_type] = hook
         else:
             raise ValueError(f"Invalid hook type: {hook_type}")
-
+    
     def execute_hook(self, hook_type: str, *args):
         hook = self.hooks.get(hook_type)
         if hook:
@@ -172,9 +170,7 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
                 if isinstance(result, webdriver.Chrome):
                     return result
                 else:
-                    raise TypeError(
-                        f"Hook {hook_type} must return an instance of webdriver.Chrome or None."
-                    )
+                    raise TypeError(f"Hook {hook_type} must return an instance of webdriver.Chrome or None.")
         # If the hook returns None or there is no hook, return self.driver
         return self.driver
 
@@ -182,77 +178,60 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         self.options.add_argument(f"user-agent={user_agent}")
         self.driver.quit()
         self.driver = webdriver.Chrome(service=self.service, options=self.options)
-        self.driver = self.execute_hook("on_user_agent_updated", self.driver)
+        self.driver = self.execute_hook('on_user_agent_updated', self.driver)
 
     def set_custom_headers(self, headers: dict):
         # Enable Network domain for sending headers
-        self.driver.execute_cdp_cmd("Network.enable", {})
+        self.driver.execute_cdp_cmd('Network.enable', {})
         # Set extra HTTP headers
-        self.driver.execute_cdp_cmd("Network.setExtraHTTPHeaders", {"headers": headers})
+        self.driver.execute_cdp_cmd('Network.setExtraHTTPHeaders', {'headers': headers})
 
-    def _ensure_page_load(self, max_checks=6, check_interval=0.01):
+    def _ensure_page_load(self,  max_checks=6, check_interval=0.01):
         initial_length = len(self.driver.page_source)
-
+        
         for ix in range(max_checks):
             # print(f"Checking page load: {ix}")
             time.sleep(check_interval)
             current_length = len(self.driver.page_source)
-
+            
             if current_length != initial_length:
                 break
 
         return self.driver.page_source
-
+    
     def crawl(self, url: str, **kwargs) -> str:
         # Create md5 hash of the URL
         import hashlib
-
         url_hash = hashlib.md5(url.encode()).hexdigest()
-
+        
         if self.use_cached_html:
-            cache_file_path = os.path.join(
-                os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()),
-                ".crawl4ai",
-                "cache",
-                url_hash,
-            )
+            cache_file_path = os.path.join(Path.home(), ".crawl4ai", "cache", url_hash)
             if os.path.exists(cache_file_path):
                 with open(cache_file_path, "r") as f:
                     return sanitize_input_encode(f.read())
 
         try:
-            self.driver = self.execute_hook("before_get_url", self.driver)
+            self.driver = self.execute_hook('before_get_url', self.driver)
             if self.verbose:
                 print(f"[LOG] 🕸️ Crawling {url} using LocalSeleniumCrawlerStrategy...")
-            self.driver.get(url)  # <html><head></head><body></body></html>
-
+            self.driver.get(url) #<html><head></head><body></body></html>
+            
             WebDriverWait(self.driver, 20).until(
-                lambda d: d.execute_script("return document.readyState") == "complete"
+                lambda d: d.execute_script('return document.readyState') == 'complete'
             )
             WebDriverWait(self.driver, 10).until(
                 EC.presence_of_all_elements_located((By.TAG_NAME, "body"))
             )
-
-            self.driver.execute_script(
-                "window.scrollTo(0, document.body.scrollHeight);"
-            )
-
-            self.driver = self.execute_hook("after_get_url", self.driver)
-            html = sanitize_input_encode(
-                self._ensure_page_load()
-            )  # self.driver.page_source
-            can_not_be_done_headless = (
-                False  # Look at my creativity for naming variables
-            )
-
+            
+            self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            
+            self.driver = self.execute_hook('after_get_url', self.driver)
+            html = sanitize_input_encode(self._ensure_page_load()) # self.driver.page_source                                        
+            can_not_be_done_headless = False # Look at my creativity for naming variables
+            
             # TODO: Very ugly approach, but promise to change it!
-            if (
-                kwargs.get("bypass_headless", False)
-                or html == "<html><head></head><body></body></html>"
-            ):
-                print(
-                    "[LOG] 🙌 Page could not be loaded in headless mode. Trying non-headless mode..."
-                )
+            if kwargs.get('bypass_headless', False) or html == "<html><head></head><body></body></html>":
+                print("[LOG] 🙌 Page could not be loaded in headless mode. Trying non-headless mode...")
                 can_not_be_done_headless = True
                 options = Options()
                 options.headless = False
@@ -260,31 +239,27 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
                 options.add_argument("--window-size=5,5")
                 driver = webdriver.Chrome(service=self.service, options=options)
                 driver.get(url)
-                self.driver = self.execute_hook("after_get_url", driver)
+                self.driver = self.execute_hook('after_get_url', driver)
                 html = sanitize_input_encode(driver.page_source)
                 driver.quit()
-
+            
             # Execute JS code if provided
             self.js_code = kwargs.get("js_code", self.js_code)
             if self.js_code and type(self.js_code) == str:
                 self.driver.execute_script(self.js_code)
                 # Optionally, wait for some condition after executing the JS code
                 WebDriverWait(self.driver, 10).until(
-                    lambda driver: driver.execute_script("return document.readyState")
-                    == "complete"
+                    lambda driver: driver.execute_script("return document.readyState") == "complete"
                 )
             elif self.js_code and type(self.js_code) == list:
                 for js in self.js_code:
                     self.driver.execute_script(js)
                     WebDriverWait(self.driver, 10).until(
-                        lambda driver: driver.execute_script(
-                            "return document.readyState"
-                        )
-                        == "complete"
+                        lambda driver: driver.execute_script("return document.readyState") == "complete"
                     )
-
+            
             # Optionally, wait for some condition after executing the JS code : Contributed by (https://github.com/jonymusky)
-            wait_for = kwargs.get("wait_for", False)
+            wait_for = kwargs.get('wait_for', False)
             if wait_for:
                 if callable(wait_for):
                     print("[LOG] 🔄 Waiting for condition...")
@@ -293,37 +268,32 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
                     print("[LOG] 🔄 Waiting for condition...")
                     WebDriverWait(self.driver, 20).until(
                         EC.presence_of_element_located((By.CSS_SELECTOR, wait_for))
-                    )
-
+                    ) 
+            
             if not can_not_be_done_headless:
                 html = sanitize_input_encode(self.driver.page_source)
-            self.driver = self.execute_hook("before_return_html", self.driver, html)
-
+            self.driver = self.execute_hook('before_return_html', self.driver, html)
+            
             # Store in cache
-            cache_file_path = os.path.join(
-                os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()),
-                ".crawl4ai",
-                "cache",
-                url_hash,
-            )
+            cache_file_path = os.path.join(Path.home(), ".crawl4ai", "cache", url_hash)
             with open(cache_file_path, "w", encoding="utf-8") as f:
                 f.write(html)
-
+                
             if self.verbose:
                 print(f"[LOG] ✅ Crawled {url} successfully!")
-
+            
             return html
-        except InvalidArgumentException as e:
-            if not hasattr(e, "msg"):
+        except InvalidArgumentException:
+            if not hasattr(e, 'msg'):
                 e.msg = sanitize_input_encode(str(e))
             raise InvalidArgumentException(f"Failed to crawl {url}: {e.msg}")
         except WebDriverException as e:
             # If e does nlt have msg attribute create it and set it to str(e)
-            if not hasattr(e, "msg"):
+            if not hasattr(e, 'msg'):
                 e.msg = sanitize_input_encode(str(e))
-            raise WebDriverException(f"Failed to crawl {url}: {e.msg}")
+            raise WebDriverException(f"Failed to crawl {url}: {e.msg}")  
         except Exception as e:
-            if not hasattr(e, "msg"):
+            if not hasattr(e, 'msg'):
                 e.msg = sanitize_input_encode(str(e))
             raise Exception(f"Failed to crawl {url}: {e.msg}")
 
@@ -331,9 +301,7 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
         try:
             # Get the dimensions of the page
             total_width = self.driver.execute_script("return document.body.scrollWidth")
-            total_height = self.driver.execute_script(
-                "return document.body.scrollHeight"
-            )
+            total_height = self.driver.execute_script("return document.body.scrollHeight")
 
             # Set the window size to the dimensions of the page
             self.driver.set_window_size(total_width, total_height)
@@ -345,27 +313,25 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
             image = Image.open(BytesIO(screenshot))
 
             # Convert image to RGB mode (this will handle both RGB and RGBA images)
-            rgb_image = image.convert("RGB")
+            rgb_image = image.convert('RGB')
 
             # Convert to JPEG and compress
             buffered = BytesIO()
             rgb_image.save(buffered, format="JPEG", quality=85)
-            img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
+            img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
 
             if self.verbose:
-                print("[LOG] 📸 Screenshot taken and converted to base64")
+                print(f"[LOG] 📸 Screenshot taken and converted to base64")
 
             return img_base64
         except Exception as e:
-            error_message = sanitize_input_encode(
-                f"Failed to take screenshot: {str(e)}"
-            )
+            error_message = sanitize_input_encode(f"Failed to take screenshot: {str(e)}")
             print(error_message)
 
             # Generate an image with black background
-            img = Image.new("RGB", (800, 600), color="black")
+            img = Image.new('RGB', (800, 600), color='black')
             draw = ImageDraw.Draw(img)
-
+            
             # Load a font
             try:
                 font = ImageFont.truetype("arial.ttf", 40)
@@ -379,16 +345,16 @@ class LocalSeleniumCrawlerStrategy(CrawlerStrategy):
 
             # Calculate text position
             text_position = (10, 10)
-
+            
             # Draw the text on the image
             draw.text(text_position, wrapped_text, fill=text_color, font=font)
-
+            
             # Convert to base64
             buffered = BytesIO()
             img.save(buffered, format="JPEG")
-            img_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
+            img_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
 
             return img_base64
-
+        
     def quit(self):
         self.driver.quit()

crawl4ai/crawlers/amazon_product/crawler.py

@@ -1,20 +0,0 @@
-from crawl4ai.hub import BaseCrawler
-
-__meta__ = {
-    "version": "1.2.0",
-    "tested_on": ["amazon.com"],
-    "rate_limit": "50 RPM",
-    "schema": {"product": ["name", "price"]}
-}
-
-class AmazonProductCrawler(BaseCrawler):
-    async def run(self, url: str, **kwargs) -> str:
-        try:
-            self.logger.info(f"Crawling {url}")
-            return '{"product": {"name": "Test Amazon Product"}}'
-        except Exception as e:
-            self.logger.error(f"Crawl failed: {str(e)}")
-            return json.dumps({
-                "error": str(e),
-                "metadata": self.meta  # Include meta in error response
-            })            

crawl4ai/crawlers/google_search/crawler.py

@@ -1,131 +0,0 @@
-from crawl4ai import BrowserConfig, AsyncWebCrawler, CrawlerRunConfig, CacheMode
-from crawl4ai.hub import BaseCrawler
-from crawl4ai.utils import optimize_html, get_home_folder, preprocess_html_for_schema
-from crawl4ai import JsonCssExtractionStrategy
-from pathlib import Path
-import json
-import os
-from typing import Dict
-
-
-class GoogleSearchCrawler(BaseCrawler):
-    __meta__ = {
-        "version": "1.0.0",
-        "tested_on": ["google.com/search*"],
-        "rate_limit": "10 RPM",
-        "description": "Crawls Google Search results (text + images)",
-    }
-
-    def __init__(self):
-        super().__init__()
-        self.js_script = (Path(__file__).parent /
-                          "script.js").read_text()
-
-    async def run(self, url="", query: str = "", search_type: str = "text", schema_cache_path = None, **kwargs) -> str:
-        """Crawl Google Search results for a query"""
-        url = f"https://www.google.com/search?q={query}&gl=sg&hl=en" if search_type == "text" else f"https://www.google.com/search?q={query}&gl=sg&hl=en&tbs=qdr:d&udm=2"
-        if kwargs.get("page_start", 1) > 1:
-            url = f"{url}&start={kwargs['page_start'] * 10}"
-        if kwargs.get("page_length", 1) > 1:
-            url = f"{url}&num={kwargs['page_length']}"
-            
-        browser_config = BrowserConfig(headless=True, verbose=True)
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            config = CrawlerRunConfig(
-                cache_mode=kwargs.get("cache_mode", CacheMode.BYPASS),
-                keep_attrs=["id", "class"],
-                keep_data_attributes=True,
-                delay_before_return_html=kwargs.get(
-                    "delay", 2 if search_type == "image" else 1),
-                js_code=self.js_script if search_type == "image" else None,
-            )
-
-            result = await crawler.arun(url=url, config=config)
-            if not result.success:
-                return json.dumps({"error": result.error})
-
-            if search_type == "image":
-                if result.js_execution_result.get("success", False) is False:
-                    return json.dumps({"error": result.js_execution_result.get("error", "Unknown error")})
-                if "results" in result.js_execution_result:
-                    image_result = result.js_execution_result['results'][0]
-                    if image_result.get("success", False) is False:
-                        return json.dumps({"error": image_result.get("error", "Unknown error")})
-                    return json.dumps(image_result["result"], indent=4)
-
-            # For text search, extract structured data
-            schemas = await self._build_schemas(result.cleaned_html, schema_cache_path)
-            extracted = {
-                key: JsonCssExtractionStrategy(schema=schemas[key]).run(
-                    url=url, sections=[result.html]
-                )
-                for key in schemas
-            }
-            return json.dumps(extracted, indent=4)
-
-    async def _build_schemas(self, html: str, schema_cache_path: str = None) -> Dict[str, Dict]:
-        """Build extraction schemas (organic, top stories, etc.)"""
-        home_dir = get_home_folder() if not schema_cache_path else schema_cache_path
-        os.makedirs(f"{home_dir}/schema", exist_ok=True)
-
-        # cleaned_html = optimize_html(html, threshold=100)
-        cleaned_html = preprocess_html_for_schema(html) 
-
-        organic_schema = None
-        if os.path.exists(f"{home_dir}/schema/organic_schema.json"):
-            with open(f"{home_dir}/schema/organic_schema.json", "r") as f:
-                organic_schema = json.load(f)
-        else:
-            organic_schema = JsonCssExtractionStrategy.generate_schema(
-                html=cleaned_html,
-                target_json_example="""{
-            "title": "...",
-            "link": "...",
-            "snippet": "...",
-            "date": "1 hour ago",
-        }""",
-                query="""The given html is the crawled html from Google search result. Please find the schema for organic search item in the given html, I am interested in title, link, snippet text. date."""
-            )
-
-            with open(f"{home_dir}/schema/organic_schema.json", "w") as f:
-                f.write(json.dumps(organic_schema))
-
-        top_stories_schema = None
-        if os.path.exists(f"{home_dir}/schema/top_stories_schema.json"):
-            with open(f"{home_dir}/schema/top_stories_schema.json", "r") as f:
-                top_stories_schema = json.load(f)
-        else:
-            top_stories_schema = JsonCssExtractionStrategy.generate_schema(
-                html=cleaned_html,
-                target_json_example="""{
-            "title": "...",
-            "link": "...",
-            "source": "Insider Monkey",
-            "date": "1 hour ago",
-        }""",
-                query="""The given html is the crawled html from Google search result. Please find the schema for Top Story item int he given html, I am interested in title, link, source. date and imageUrl."""
-            )
-
-            with open(f"{home_dir}/schema/top_stories_schema.json", "w") as f:
-                f.write(json.dumps(top_stories_schema))
-
-        suggested_query_schema = None
-        if os.path.exists(f"{home_dir}/schema/suggested_query_schema.json"):
-            with open(f"{home_dir}/schema/suggested_query_schema.json", "r") as f:
-                suggested_query_schema = json.load(f)
-        else:
-            suggested_query_schema = JsonCssExtractionStrategy.generate_schema(
-                html=cleaned_html,
-                target_json_example="""{
-            "query": "A for Apple",
-        }""",
-                query="""The given HTML contains the crawled HTML from Google search results. Please find the schema for each suggested query in the section "People also search for" within the given HTML. I am interested in the queries only."""
-            )
-            with open(f"{home_dir}/schema/suggested_query_schema.json", "w") as f:
-                f.write(json.dumps(suggested_query_schema))
-
-        return {
-            "organic_schema": organic_schema,
-            "top_stories_schema": top_stories_schema,
-            "suggested_query_schema": suggested_query_schema,
-        }

crawl4ai/crawlers/google_search/script.js

@@ -1,115 +0,0 @@
-(() => {
-    // Function to extract image data from Google Images page
-    function extractImageData() {
-        const keys = Object.keys(window.W_jd);
-        let allImageData = [];
-        let currentPosition = 0;
-
-        // Get the symbol we'll use (from first valid entry)
-        let targetSymbol;
-        for (let key of keys) {
-            try {
-                const symbols = Object.getOwnPropertySymbols(window.W_jd[key]);
-                if (symbols.length > 0) {
-                    targetSymbol = symbols[0];
-                    break;
-                }
-            } catch (e) {
-                continue;
-            }
-        }
-
-        if (!targetSymbol) return [];
-
-        // Iterate through ALL keys
-        for (let key of keys) {
-            try {
-                const o1 = window.W_jd[key][targetSymbol]
-                if (!o1) continue;
-                const data = Object.values(o1)[0]
-                // const data = window.W_jd[key][targetSymbol]?.Ws;
-                // Check if this is a valid image data entry
-                if (data && Array.isArray(data[1])) {
-                    const processedData = processImageEntry(data, currentPosition);
-                    if (processedData) {
-                        allImageData.push(processedData);
-                        currentPosition++;
-                    }
-                }
-            } catch (e) {
-                continue;
-            }
-        }
-
-        return allImageData;
-    }
-
-    function processImageEntry(entry, position) {
-        const imageData = entry[1];
-        if (!Array.isArray(imageData)) return null;
-
-        // Extract the image ID
-        const imageId = imageData[1];
-        if (!imageId) return null;
-
-        // Find the corresponding DOM element
-        const domElement = document.querySelector(`[data-docid="${imageId}"]`);
-        if (!domElement) return null;
-
-        // Extract data from the array structure
-        const [
-            _,
-            id,
-            thumbnailInfo,
-            imageInfo,
-            __,
-            ___,
-            rgb,
-            ____,
-            _____,
-            metadata
-        ] = imageData;
-
-        // Ensure we have the required data
-        if (!thumbnailInfo || !imageInfo) return null;
-
-        // Extract metadata from DOM
-        const title = domElement?.querySelector('.toI8Rb')?.textContent?.trim();
-        const source = domElement?.querySelector('.guK3rf')?.textContent?.trim();
-        const link = domElement?.querySelector('a.EZAeBe')?.href;
-
-        if (!link) return null;
-
-        // Build Google Image URL
-        const googleUrl = buildGoogleImageUrl(imageInfo[0], link, imageId, imageInfo[1], imageInfo[2]);
-
-        return {
-            title,
-            imageUrl: imageInfo[0],
-            imageWidth: imageInfo[2],
-            imageHeight: imageInfo[1],
-            thumbnailUrl: thumbnailInfo[0],
-            thumbnailWidth: thumbnailInfo[2],
-            thumbnailHeight: thumbnailInfo[1],
-            source,
-            domain: metadata['2000']?.[1] || new URL(link).hostname,
-            link,
-            googleUrl,
-            position: position + 1
-        };
-    }
-
-    function buildGoogleImageUrl(imgUrl, refUrl, tbnid, height, width) {
-        const params = new URLSearchParams({
-            imgurl: imgUrl,
-            tbnid: tbnid,
-            imgrefurl: refUrl,
-            docid: tbnid,
-            w: width.toString(),
-            h: height.toString(),
-        });
-
-        return `https://www.google.com/imgres?${params.toString()}`;
-    }
-    return extractImageData();
-})();

crawl4ai/database.py

@@ -3,17 +3,15 @@ from pathlib import Path
 import sqlite3
 from typing import Optional, Tuple
 
-DB_PATH = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+DB_PATH = os.path.join(Path.home(), ".crawl4ai")
 os.makedirs(DB_PATH, exist_ok=True)
 DB_PATH = os.path.join(DB_PATH, "crawl4ai.db")
 
-
 def init_db():
     global DB_PATH
     conn = sqlite3.connect(DB_PATH)
     cursor = conn.cursor()
-    cursor.execute(
-        """
+    cursor.execute('''
         CREATE TABLE IF NOT EXISTS crawled_data (
             url TEXT PRIMARY KEY,
             html TEXT,
@@ -26,42 +24,31 @@ def init_db():
             metadata TEXT DEFAULT "{}",
             screenshot TEXT DEFAULT ""
         )
-    """
-    )
+    ''')
     conn.commit()
     conn.close()
 
-
 def alter_db_add_screenshot(new_column: str = "media"):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute(
-            f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""'
-        )
+        cursor.execute(f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""')
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error altering database to add screenshot column: {e}")
 
-
 def check_db_path():
     if not DB_PATH:
         raise ValueError("Database path is not set or is empty.")
 
-
-def get_cached_url(
-    url: str,
-) -> Optional[Tuple[str, str, str, str, str, str, str, bool, str]]:
+def get_cached_url(url: str) -> Optional[Tuple[str, str, str, str, str, str, str, bool, str]]:
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute(
-            "SELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?",
-            (url,),
-        )
+        cursor.execute('SELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?', (url,))
         result = cursor.fetchone()
         conn.close()
         return result
@@ -69,25 +56,12 @@ def get_cached_url(
         print(f"Error retrieving cached URL: {e}")
         return None
 
-
-def cache_url(
-    url: str,
-    html: str,
-    cleaned_html: str,
-    markdown: str,
-    extracted_content: str,
-    success: bool,
-    media: str = "{}",
-    links: str = "{}",
-    metadata: str = "{}",
-    screenshot: str = "",
-):
+def cache_url(url: str, html: str, cleaned_html: str, markdown: str, extracted_content: str, success: bool, media : str = "{}", links : str = "{}", metadata : str = "{}", screenshot: str = ""):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute(
-            """
+        cursor.execute('''
             INSERT INTO crawled_data (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot)
             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ON CONFLICT(url) DO UPDATE SET
@@ -100,32 +74,18 @@ def cache_url(
                 links = excluded.links,    
                 metadata = excluded.metadata,      
                 screenshot = excluded.screenshot
-        """,
-            (
-                url,
-                html,
-                cleaned_html,
-                markdown,
-                extracted_content,
-                success,
-                media,
-                links,
-                metadata,
-                screenshot,
-            ),
-        )
+        ''', (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot))
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error caching URL: {e}")
 
-
 def get_total_count() -> int:
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute("SELECT COUNT(*) FROM crawled_data")
+        cursor.execute('SELECT COUNT(*) FROM crawled_data')
         result = cursor.fetchone()
         conn.close()
         return result[0]
@@ -133,48 +93,43 @@ def get_total_count() -> int:
         print(f"Error getting total count: {e}")
         return 0
 
-
 def clear_db():
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute("DELETE FROM crawled_data")
+        cursor.execute('DELETE FROM crawled_data')
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error clearing database: {e}")
-
-
+        
 def flush_db():
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute("DROP TABLE crawled_data")
+        cursor.execute('DROP TABLE crawled_data')
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error flushing database: {e}")
 
-
 def update_existing_records(new_column: str = "media", default_value: str = "{}"):
     check_db_path()
     try:
         conn = sqlite3.connect(DB_PATH)
         cursor = conn.cursor()
-        cursor.execute(
-            f'UPDATE crawled_data SET {new_column} = "{default_value}" WHERE screenshot IS NULL'
-        )
+        cursor.execute(f'UPDATE crawled_data SET {new_column} = "{default_value}" WHERE screenshot IS NULL')
         conn.commit()
         conn.close()
     except Exception as e:
         print(f"Error updating existing records: {e}")
 
-
 if __name__ == "__main__":
     # Delete the existing database file
     if os.path.exists(DB_PATH):
         os.remove(DB_PATH)
-    init_db()
+    init_db()  
     # alter_db_add_screenshot("COL_NAME")
+    

crawl4ai/deep_crawling/__init__.py

@@ -1,47 +0,0 @@
-# deep_crawling/__init__.py
-from .base_strategy import DeepCrawlDecorator, DeepCrawlStrategy
-from .bfs_strategy import BFSDeepCrawlStrategy
-from .bff_strategy import BestFirstCrawlingStrategy
-from .dfs_strategy import DFSDeepCrawlStrategy
-from .filters import (
-    FilterChain,
-    ContentTypeFilter,
-    DomainFilter,
-    URLFilter,
-    URLPatternFilter,
-    FilterStats,
-    ContentRelevanceFilter,
-    SEOFilter
-)
-from .scorers import (
-    KeywordRelevanceScorer,
-    URLScorer,
-    CompositeScorer,
-    DomainAuthorityScorer,
-    FreshnessScorer,
-    PathDepthScorer,
-    ContentTypeScorer
-)
-
-__all__ = [
-    "DeepCrawlDecorator",
-    "DeepCrawlStrategy",
-    "BFSDeepCrawlStrategy",
-    "BestFirstCrawlingStrategy",
-    "DFSDeepCrawlStrategy",
-    "FilterChain",
-    "ContentTypeFilter",
-    "DomainFilter",
-    "URLFilter",
-    "URLPatternFilter",
-    "FilterStats",
-    "ContentRelevanceFilter",
-    "SEOFilter",
-    "KeywordRelevanceScorer",
-    "URLScorer",
-    "CompositeScorer",
-    "DomainAuthorityScorer",
-    "FreshnessScorer",
-    "PathDepthScorer",
-    "ContentTypeScorer",
-]

crawl4ai/deep_crawling/base_strategy.py

@@ -1,159 +0,0 @@
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from typing import AsyncGenerator, Optional, Set, List, Dict
-from functools import wraps
-from contextvars import ContextVar
-from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
-
-
-class DeepCrawlDecorator:
-    """Decorator that adds deep crawling capability to arun method."""
-    deep_crawl_active = ContextVar("deep_crawl_active", default=False)
-    
-    def __init__(self, crawler: AsyncWebCrawler): 
-        self.crawler = crawler
-
-    def __call__(self, original_arun):
-        @wraps(original_arun)
-        async def wrapped_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
-            # If deep crawling is already active, call the original method to avoid recursion.
-            if config and config.deep_crawl_strategy and not self.deep_crawl_active.get():
-                token = self.deep_crawl_active.set(True)
-                # Await the arun call to get the actual result object.
-                result_obj = await config.deep_crawl_strategy.arun(
-                    crawler=self.crawler,
-                    start_url=url,
-                    config=config
-                )
-                if config.stream:
-                    async def result_wrapper():
-                        try:
-                            async for result in result_obj:
-                                yield result
-                        finally:
-                            self.deep_crawl_active.reset(token)
-                    return result_wrapper()
-                else:
-                    try:
-                        return result_obj
-                    finally:
-                        self.deep_crawl_active.reset(token)
-            return await original_arun(url, config=config, **kwargs)
-        return wrapped_arun
-
-class DeepCrawlStrategy(ABC):
-    """
-    Abstract base class for deep crawling strategies.
-    
-    Core functions:
-      - arun: Main entry point that returns an async generator of CrawlResults.
-      - shutdown: Clean up resources.
-      - can_process_url: Validate a URL and decide whether to process it.
-      - _process_links: Extract and process links from a CrawlResult.
-    """
-
-    @abstractmethod
-    async def _arun_batch(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
-        """
-        Batch (non-streaming) mode:
-        Processes one BFS level at a time, then yields all the results.
-        """
-        pass
-
-    @abstractmethod
-    async def _arun_stream(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
-        """
-        Streaming mode:
-        Processes one BFS level at a time and yields results immediately as they arrive.
-        """
-        pass
-    
-    async def arun(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: Optional[CrawlerRunConfig] = None,
-    ) -> RunManyReturn:
-        """
-        Traverse the given URL using the specified crawler.
-        
-        Args:
-            start_url (str): The URL from which to start crawling.
-            crawler (AsyncWebCrawler): The crawler instance to use.
-            crawler_run_config (Optional[CrawlerRunConfig]): Crawler configuration.
-        
-        Returns:
-            Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
-        """
-        if config is None:
-            raise ValueError("CrawlerRunConfig must be provided")
-
-        if config.stream:
-            return self._arun_stream(start_url, crawler, config)
-        else:
-            return await self._arun_batch(start_url, crawler, config)
-
-    def __call__(self, start_url: str, crawler: AsyncWebCrawler, config: CrawlerRunConfig):
-        return self.arun(start_url, crawler, config)
-
-    @abstractmethod
-    async def shutdown(self) -> None:
-        """
-        Clean up resources used by the deep crawl strategy.
-        """
-        pass
-
-    @abstractmethod
-    async def can_process_url(self, url: str, depth: int) -> bool:
-        """
-        Validate the URL format and apply custom filtering logic.
-        
-        Args:
-            url (str): The URL to validate.
-            depth (int): The current depth in the crawl.
-        
-        Returns:
-            bool: True if the URL should be processed, False otherwise.
-        """
-        pass
-
-    @abstractmethod
-    async def link_discovery(
-        self,
-        result: CrawlResult,
-        source_url: str,
-        current_depth: int,
-        visited: Set[str],
-        next_level: List[tuple],
-        depths: Dict[str, int],
-    ) -> None:
-        """
-        Extract and process links from the given crawl result.
-        
-        This method should:
-          - Validate each extracted URL using can_process_url.
-          - Optionally score URLs.
-          - Append valid URLs (and their parent references) to the next_level list.
-          - Update the depths dictionary with the new depth for each URL.
-        
-        Args:
-            result (CrawlResult): The result from a crawl operation.
-            source_url (str): The URL from which this result was obtained.
-            current_depth (int): The depth at which the source URL was processed.
-            visited (Set[str]): Set of already visited URLs.
-            next_level (List[tuple]): List of tuples (url, parent_url) for the next BFS level.
-            depths (Dict[str, int]): Mapping of URLs to their current depth.
-        """
-        pass
-

crawl4ai/deep_crawling/bff_strategy.py

@@ -1,271 +0,0 @@
-# best_first_crawling_strategy.py
-import asyncio
-import logging
-from datetime import datetime
-from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
-from urllib.parse import urlparse
-
-from ..models import TraversalStats
-from .filters import FilterChain
-from .scorers import URLScorer
-from . import DeepCrawlStrategy
-
-from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult, RunManyReturn
-from ..utils import normalize_url_for_deep_crawl
-
-from math import inf as infinity
-
-# Configurable batch size for processing items from the priority queue
-BATCH_SIZE = 10
-
-
-class BestFirstCrawlingStrategy(DeepCrawlStrategy):
-    """
-    Best-First Crawling Strategy using a priority queue.
-    
-    This strategy prioritizes URLs based on their score, ensuring that higher-value
-    pages are crawled first. It reimplements the core traversal loop to use a priority
-    queue while keeping URL validation and link discovery consistent with our design.
-    
-    Core methods:
-      - arun: Returns either a list (batch mode) or an async generator (stream mode).
-      - _arun_best_first: Core generator that uses a priority queue to yield CrawlResults.
-      - can_process_url: Validates URLs and applies filtering (inherited behavior).
-      - link_discovery: Extracts and validates links from a CrawlResult.
-    """
-    def __init__(
-        self,
-        max_depth: int,
-        filter_chain: FilterChain = FilterChain(),
-        url_scorer: Optional[URLScorer] = None,
-        include_external: bool = False,
-        max_pages: int = infinity,
-        logger: Optional[logging.Logger] = None,
-    ):
-        self.max_depth = max_depth
-        self.filter_chain = filter_chain
-        self.url_scorer = url_scorer
-        self.include_external = include_external
-        self.max_pages = max_pages
-        # self.logger = logger or logging.getLogger(__name__)
-        # Ensure logger is always a Logger instance, not a dict from serialization
-        if isinstance(logger, logging.Logger):
-            self.logger = logger
-        else:
-            # Create a new logger if logger is None, dict, or any other non-Logger type
-            self.logger = logging.getLogger(__name__)
-        self.stats = TraversalStats(start_time=datetime.now())
-        self._cancel_event = asyncio.Event()
-        self._pages_crawled = 0
-
-    async def can_process_url(self, url: str, depth: int) -> bool:
-        """
-        Validate the URL format and apply filtering.
-        For the starting URL (depth 0), filtering is bypassed.
-        """
-        try:
-            parsed = urlparse(url)
-            if not parsed.scheme or not parsed.netloc:
-                raise ValueError("Missing scheme or netloc")
-            if parsed.scheme not in ("http", "https"):
-                raise ValueError("Invalid scheme")
-            if "." not in parsed.netloc:
-                raise ValueError("Invalid domain")
-        except Exception as e:
-            self.logger.warning(f"Invalid URL: {url}, error: {e}")
-            return False
-
-        if depth != 0 and not await self.filter_chain.apply(url):
-            return False
-
-        return True
-
-    async def link_discovery(
-        self,
-        result: CrawlResult,
-        source_url: str,
-        current_depth: int,
-        visited: Set[str],
-        next_links: List[Tuple[str, Optional[str]]],
-        depths: Dict[str, int],
-    ) -> None:
-        """
-        Extract links from the crawl result, validate them, and append new URLs
-        (with their parent references) to next_links.
-        Also updates the depths dictionary.
-        """
-        new_depth = current_depth + 1
-        if new_depth > self.max_depth:
-            return
-            
-        # If we've reached the max pages limit, don't discover new links
-        remaining_capacity = self.max_pages - self._pages_crawled
-        if remaining_capacity <= 0:
-            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
-            return
-
-        # Retrieve internal links; include external links if enabled.
-        links = result.links.get("internal", [])
-        if self.include_external:
-            links += result.links.get("external", [])
-
-        # If we have more links than remaining capacity, limit how many we'll process
-        valid_links = []
-        for link in links:
-            url = link.get("href")
-            base_url = normalize_url_for_deep_crawl(url, source_url)
-            if base_url in visited:
-                continue
-            if not await self.can_process_url(url, new_depth):
-                self.stats.urls_skipped += 1
-                continue
-                
-            valid_links.append(base_url)
-            
-        # Record the new depths and add to next_links
-        for url in valid_links:
-            depths[url] = new_depth
-            next_links.append((url, source_url))
-
-    async def _arun_best_first(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
-        """
-        Core best-first crawl method using a priority queue.
-        
-        The queue items are tuples of (score, depth, url, parent_url). Lower scores
-        are treated as higher priority. URLs are processed in batches for efficiency.
-        """
-        queue: asyncio.PriorityQueue = asyncio.PriorityQueue()
-        # Push the initial URL with score 0 and depth 0.
-        initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0
-        await queue.put((-initial_score, 0, start_url, None))
-        visited: Set[str] = set()
-        depths: Dict[str, int] = {start_url: 0}
-
-        while not queue.empty() and not self._cancel_event.is_set():
-            # Stop if we've reached the max pages limit
-            if self._pages_crawled >= self.max_pages:
-                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
-                break
-                
-            # Calculate how many more URLs we can process in this batch
-            remaining = self.max_pages - self._pages_crawled
-            batch_size = min(BATCH_SIZE, remaining)
-            if batch_size <= 0:
-                # No more pages to crawl
-                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
-                break
-                
-            batch: List[Tuple[float, int, str, Optional[str]]] = []
-            # Retrieve up to BATCH_SIZE items from the priority queue.
-            for _ in range(BATCH_SIZE):
-                if queue.empty():
-                    break
-                item = await queue.get()
-                score, depth, url, parent_url = item
-                if url in visited:
-                    continue
-                visited.add(url)
-                batch.append(item)
-
-            if not batch:
-                continue
-
-            # Process the current batch of URLs.
-            urls = [item[2] for item in batch]
-            batch_config = config.clone(deep_crawl_strategy=None, stream=True)
-            stream_gen = await crawler.arun_many(urls=urls, config=batch_config)
-            async for result in stream_gen:
-                result_url = result.url
-                # Find the corresponding tuple from the batch.
-                corresponding = next((item for item in batch if item[2] == result_url), None)
-                if not corresponding:
-                    continue
-                score, depth, url, parent_url = corresponding
-                result.metadata = result.metadata or {}
-                result.metadata["depth"] = depth
-                result.metadata["parent_url"] = parent_url
-                result.metadata["score"] = -score
-                
-                # Count only successful crawls toward max_pages limit
-                if result.success:
-                    self._pages_crawled += 1
-                    # Check if we've reached the limit during batch processing
-                    if self._pages_crawled >= self.max_pages:
-                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
-                        break  # Exit the generator
-                
-                yield result
-                
-                # Only discover links from successful crawls
-                if result.success:
-                    # Discover new links from this result
-                    new_links: List[Tuple[str, Optional[str]]] = []
-                    await self.link_discovery(result, result_url, depth, visited, new_links, depths)
-                    
-                    for new_url, new_parent in new_links:
-                        new_depth = depths.get(new_url, depth + 1)
-                        new_score = self.url_scorer.score(new_url) if self.url_scorer else 0
-                        await queue.put((-new_score, new_depth, new_url, new_parent))
-
-        # End of crawl.
-
-    async def _arun_batch(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
-        """
-        Best-first crawl in batch mode.
-        
-        Aggregates all CrawlResults into a list.
-        """
-        results: List[CrawlResult] = []
-        async for result in self._arun_best_first(start_url, crawler, config):
-            results.append(result)
-        return results
-
-    async def _arun_stream(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
-        """
-        Best-first crawl in streaming mode.
-        
-        Yields CrawlResults as they become available.
-        """
-        async for result in self._arun_best_first(start_url, crawler, config):
-            yield result
-
-    async def arun(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: Optional[CrawlerRunConfig] = None,
-    ) -> "RunManyReturn":
-        """
-        Main entry point for best-first crawling.
-        
-        Returns either a list (batch mode) or an async generator (stream mode)
-        of CrawlResults.
-        """
-        if config is None:
-            raise ValueError("CrawlerRunConfig must be provided")
-        if config.stream:
-            return self._arun_stream(start_url, crawler, config)
-        else:
-            return await self._arun_batch(start_url, crawler, config)
-
-    async def shutdown(self) -> None:
-        """
-        Signal cancellation and clean up resources.
-        """
-        self._cancel_event.set()
-        self.stats.end_time = datetime.now()

crawl4ai/deep_crawling/bfs_strategy.py

@@ -1,260 +0,0 @@
-# bfs_deep_crawl_strategy.py
-import asyncio
-import logging
-from datetime import datetime
-from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
-from urllib.parse import urlparse
-
-from ..models import TraversalStats
-from .filters import FilterChain
-from .scorers import URLScorer
-from . import DeepCrawlStrategy  
-from ..types import AsyncWebCrawler, CrawlerRunConfig, CrawlResult
-from ..utils import normalize_url_for_deep_crawl, efficient_normalize_url_for_deep_crawl
-from math import inf as infinity
-
-class BFSDeepCrawlStrategy(DeepCrawlStrategy):
-    """
-    Breadth-First Search deep crawling strategy.
-    
-    Core functions:
-      - arun: Main entry point; splits execution into batch or stream modes.
-      - link_discovery: Extracts, filters, and (if needed) scores the outgoing URLs.
-      - can_process_url: Validates URL format and applies the filter chain.
-    """
-    def __init__(
-        self,
-        max_depth: int,
-        filter_chain: FilterChain = FilterChain(),
-        url_scorer: Optional[URLScorer] = None,        
-        include_external: bool = False,
-        score_threshold: float = -infinity,
-        max_pages: int = infinity,
-        logger: Optional[logging.Logger] = None,
-    ):
-        self.max_depth = max_depth
-        self.filter_chain = filter_chain
-        self.url_scorer = url_scorer
-        self.include_external = include_external
-        self.score_threshold = score_threshold
-        self.max_pages = max_pages
-        # self.logger = logger or logging.getLogger(__name__)
-        # Ensure logger is always a Logger instance, not a dict from serialization
-        if isinstance(logger, logging.Logger):
-            self.logger = logger
-        else:
-            # Create a new logger if logger is None, dict, or any other non-Logger type
-            self.logger = logging.getLogger(__name__)
-        self.stats = TraversalStats(start_time=datetime.now())
-        self._cancel_event = asyncio.Event()
-        self._pages_crawled = 0
-
-    async def can_process_url(self, url: str, depth: int) -> bool:
-        """
-        Validates the URL and applies the filter chain.
-        For the start URL (depth 0) filtering is bypassed.
-        """
-        try:
-            parsed = urlparse(url)
-            if not parsed.scheme or not parsed.netloc:
-                raise ValueError("Missing scheme or netloc")
-            if parsed.scheme not in ("http", "https"):
-                raise ValueError("Invalid scheme")
-            if "." not in parsed.netloc:
-                raise ValueError("Invalid domain")
-        except Exception as e:
-            self.logger.warning(f"Invalid URL: {url}, error: {e}")
-            return False
-
-        if depth != 0 and not await self.filter_chain.apply(url):
-            return False
-
-        return True
-
-    async def link_discovery(
-        self,
-        result: CrawlResult,
-        source_url: str,
-        current_depth: int,
-        visited: Set[str],
-        next_level: List[Tuple[str, Optional[str]]],
-        depths: Dict[str, int],
-    ) -> None:
-        """
-        Extracts links from the crawl result, validates and scores them, and
-        prepares the next level of URLs.
-        Each valid URL is appended to next_level as a tuple (url, parent_url)
-        and its depth is tracked.
-        """            
-        next_depth = current_depth + 1
-        if next_depth > self.max_depth:
-            return
-
-        # If we've reached the max pages limit, don't discover new links
-        remaining_capacity = self.max_pages - self._pages_crawled
-        if remaining_capacity <= 0:
-            self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping link discovery")
-            return
-
-        # Get internal links and, if enabled, external links.
-        links = result.links.get("internal", [])
-        if self.include_external:
-            links += result.links.get("external", [])
-
-        valid_links = []
-        
-        # First collect all valid links
-        for link in links:
-            url = link.get("href")
-            # Strip URL fragments to avoid duplicate crawling
-            # base_url = url.split('#')[0] if url else url
-            base_url = normalize_url_for_deep_crawl(url, source_url)
-            if base_url in visited:
-                continue
-            if not await self.can_process_url(url, next_depth):
-                self.stats.urls_skipped += 1
-                continue
-
-            # Score the URL if a scorer is provided
-            score = self.url_scorer.score(base_url) if self.url_scorer else 0
-            
-            # Skip URLs with scores below the threshold
-            if score < self.score_threshold:
-                self.logger.debug(f"URL {url} skipped: score {score} below threshold {self.score_threshold}")
-                self.stats.urls_skipped += 1
-                continue
-
-            visited.add(base_url)
-            valid_links.append((base_url, score))
-        
-        # If we have more valid links than capacity, sort by score and take the top ones
-        if len(valid_links) > remaining_capacity:
-            if self.url_scorer:
-                # Sort by score in descending order
-                valid_links.sort(key=lambda x: x[1], reverse=True)
-            # Take only as many as we have capacity for
-            valid_links = valid_links[:remaining_capacity]
-            self.logger.info(f"Limiting to {remaining_capacity} URLs due to max_pages limit")
-            
-        # Process the final selected links
-        for url, score in valid_links:
-            # attach the score to metadata if needed
-            if score:
-                result.metadata = result.metadata or {}
-                result.metadata["score"] = score
-            next_level.append((url, source_url))
-            depths[url] = next_depth
-
-    async def _arun_batch(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
-        """
-        Batch (non-streaming) mode:
-        Processes one BFS level at a time, then yields all the results.
-        """
-        visited: Set[str] = set()
-        # current_level holds tuples: (url, parent_url)
-        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
-        depths: Dict[str, int] = {start_url: 0}
-
-        results: List[CrawlResult] = []
-
-        while current_level and not self._cancel_event.is_set():
-            # Check if we've already reached max_pages before starting a new level
-            if self._pages_crawled >= self.max_pages:
-                self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
-                break
-            
-            next_level: List[Tuple[str, Optional[str]]] = []
-            urls = [url for url, _ in current_level]
-
-            # Clone the config to disable deep crawling recursion and enforce batch mode.
-            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
-            batch_results = await crawler.arun_many(urls=urls, config=batch_config)
-            
-            # Update pages crawled counter - count only successful crawls
-            successful_results = [r for r in batch_results if r.success]
-            self._pages_crawled += len(successful_results)
-            
-            for result in batch_results:
-                url = result.url
-                depth = depths.get(url, 0)
-                result.metadata = result.metadata or {}
-                result.metadata["depth"] = depth
-                parent_url = next((parent for (u, parent) in current_level if u == url), None)
-                result.metadata["parent_url"] = parent_url
-                results.append(result)
-                
-                # Only discover links from successful crawls
-                if result.success:
-                    # Link discovery will handle the max pages limit internally
-                    await self.link_discovery(result, url, depth, visited, next_level, depths)
-
-            current_level = next_level
-
-        return results
-
-    async def _arun_stream(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
-        """
-        Streaming mode:
-        Processes one BFS level at a time and yields results immediately as they arrive.
-        """
-        visited: Set[str] = set()
-        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
-        depths: Dict[str, int] = {start_url: 0}
-
-        while current_level and not self._cancel_event.is_set():
-            next_level: List[Tuple[str, Optional[str]]] = []
-            urls = [url for url, _ in current_level]
-            visited.update(urls)
-
-            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
-            stream_gen = await crawler.arun_many(urls=urls, config=stream_config)
-            
-            # Keep track of processed results for this batch
-            results_count = 0
-            async for result in stream_gen:
-                url = result.url
-                depth = depths.get(url, 0)
-                result.metadata = result.metadata or {}
-                result.metadata["depth"] = depth
-                parent_url = next((parent for (u, parent) in current_level if u == url), None)
-                result.metadata["parent_url"] = parent_url
-                
-                # Count only successful crawls
-                if result.success:
-                    self._pages_crawled += 1
-                    # Check if we've reached the limit during batch processing
-                    if self._pages_crawled >= self.max_pages:
-                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
-                        break  # Exit the generator
-                
-                results_count += 1
-                yield result
-                
-                # Only discover links from successful crawls
-                if result.success:
-                    # Link discovery will handle the max pages limit internally
-                    await self.link_discovery(result, url, depth, visited, next_level, depths)
-            
-            # If we didn't get results back (e.g. due to errors), avoid getting stuck in an infinite loop
-            # by considering these URLs as visited but not counting them toward the max_pages limit
-            if results_count == 0 and urls:
-                self.logger.warning(f"No results returned for {len(urls)} URLs, marking as visited")
-                
-            current_level = next_level
-
-    async def shutdown(self) -> None:
-        """
-        Clean up resources and signal cancellation of the crawl.
-        """
-        self._cancel_event.set()
-        self.stats.end_time = datetime.now()

crawl4ai/deep_crawling/crazy.py

@@ -1,432 +0,0 @@
-from __future__ import annotations
-# I just got crazy, trying to wrute K&R C but in Python. Right now I feel like I'm in a quantum state.
-# I probably won't use this; I just want to leave it here. A century later, the future human race will be like, "WTF?"
-
-# ------ Imports That Will Make You Question Reality ------ #
-from functools import wraps
-from contextvars import ContextVar
-import inspect
-
-from crawl4ai import CacheMode
-from crawl4ai.async_configs import CrawlerRunConfig
-from crawl4ai.models import CrawlResult, TraversalStats
-from crawl4ai.deep_crawling.filters import FilterChain
-from crawl4ai.async_webcrawler import AsyncWebCrawler
-import time
-import logging
-from urllib.parse import urlparse
-
-from abc import ABC, abstractmethod
-from collections import deque
-import asyncio
-from typing import (
-    AsyncGenerator,
-    Dict,
-    List,
-    TypeVar,
-    Generic,
-    Tuple,
-    Callable,
-    Awaitable,
-    Union,
-)
-from functools import lru_cache
-import mmh3
-from bitarray import bitarray
-import numpy as np
-from heapq import heappush, heappop
-
-# ------ Type Algebra Mastery ------ #
-CrawlResultT = TypeVar("CrawlResultT", bound="CrawlResult")
-PriorityT = TypeVar("PriorityT")
-P = TypeVar("P")
-
-# ------ Hyperscalar Context Management ------ #
-deep_crawl_ctx = ContextVar("deep_crawl_stack", default=deque())
-
-# ------ Algebraic Crawler Monoid ------ #
-class TraversalContext:
-    __slots__ = ('visited', 'frontier', 'depths', 'priority_fn', 'current_depth')
-    
-    def __init__(self,
-                 priority_fn: Callable[[str], Awaitable[float]] = lambda _: 1.0):
-        self.visited: BloomFilter = BloomFilter(10**6, 0.01)  # 1M items, 1% FP
-        self.frontier: PriorityQueue = PriorityQueue()
-        self.depths: Dict[str, int] = {}
-        self.priority_fn = priority_fn
-        self.current_depth = 0
-
-    def clone_for_level(self) -> TraversalContext:
-        """Monadic context propagation"""
-        new_ctx = TraversalContext(self.priority_fn)
-        new_ctx.visited = self.visited.copy()
-        new_ctx.depths = self.depths.copy()
-        new_ctx.current_depth = self.current_depth
-        return new_ctx
-
-class PriorityQueue(Generic[PriorityT]):
-    """Fibonacci heap-inspired priority queue with O(1) amortized operations"""
-    __slots__ = ('_heap', '_index')
-
-    def __init__(self):
-        self._heap: List[Tuple[PriorityT, float, P]] = []
-        self._index: Dict[P, int] = {}
-
-    def insert(self, priority: PriorityT, item: P) -> None:
-        tiebreaker = time.time()  # Ensure FIFO for equal priorities
-        heappush(self._heap, (priority, tiebreaker, item))
-        self._index[item] = len(self._heap) - 1
-
-    def extract(self, top_n = 1) -> P:
-        items = []
-        for _ in range(top_n):
-            if not self._heap:
-                break
-            priority, _, item = heappop(self._heap)
-            del self._index[item]
-            items.append(item)
-        if not items:
-            raise IndexError("Priority queue empty")
-        return items
-        # while self._heap:
-        #     _, _, item = heappop(self._heap)
-        #     if item in self._index:
-        #         del self._index[item]
-        #         return item
-        raise IndexError("Priority queue empty")
-
-
-    def is_empty(self) -> bool:
-        return not bool(self._heap)
-
-class BloomFilter:
-    """Optimal Bloom filter using murmur3 hash avalanche"""
-    __slots__ = ('size', 'hashes', 'bits')
-
-    def __init__(self, capacity: int, error_rate: float):
-        self.size = self._optimal_size(capacity, error_rate)
-        self.hashes = self._optimal_hashes(capacity, self.size)
-        self.bits = bitarray(self.size)
-        self.bits.setall(False)
-
-    @staticmethod
-    def _optimal_size(n: int, p: float) -> int:
-        m = - (n * np.log(p)) / (np.log(2) ** 2)
-        return int(np.ceil(m))
-
-    @staticmethod
-    def _optimal_hashes(n: int, m: int) -> int:
-        k = (m / n) * np.log(2)
-        return int(np.ceil(k))
-
-    def add(self, item: str) -> None:
-        for seed in range(self.hashes):
-            digest = mmh3.hash(item, seed) % self.size
-            self.bits[digest] = True
-
-    def __contains__(self, item: str) -> bool:
-        return all(
-            self.bits[mmh3.hash(item, seed) % self.size]
-            for seed in range(self.hashes)
-        )
-
-    def copy(self) -> BloomFilter:
-        new = object.__new__(BloomFilter)
-        new.size = self.size
-        new.hashes = self.hashes
-        new.bits = self.bits.copy()
-        return new
-    
-    def __len__(self) -> int:
-        """
-        Estimates the number of items in the filter using the 
-        count of set bits and the formula:
-        n = -m/k * ln(1 - X/m)
-        where:
-            m = size of bit array
-            k = number of hash functions
-            X = count of set bits
-        """
-        set_bits = self.bits.count(True)
-        if set_bits == 0:
-            return 0
-            
-        # Use the inverse bloom filter formula to estimate cardinality
-        return int(
-            -(self.size / self.hashes) * 
-            np.log(1 - set_bits / self.size)
-        )
-    
-    def bit_count(self) -> int:
-        """Returns the raw count of set bits in the filter"""
-        return self.bits.count(True)
-        
-    def __repr__(self) -> str:
-        return f"BloomFilter(est_items={len(self)}, bits={self.bit_count()}/{self.size})"
-
-# ------ Hyper-Optimal Deep Crawl Core ------ #
-class DeepCrawlDecorator:
-    """Metaprogramming marvel: Zero-cost deep crawl abstraction"""
-    def __init__(self, crawler: AsyncWebCrawler):
-        self.crawler = crawler
-
-    def __call__(self, original_arun: Callable) -> Callable:
-        @wraps(original_arun)
-        async def quantum_arun(url: str, config: CrawlerRunConfig = None, **kwargs):
-            stack = deep_crawl_ctx.get()
-            if config and config.deep_crawl_strategy and not stack:
-                stack.append(self.crawler)
-                try:
-                    deep_crawl_ctx.set(stack)
-                    async for result in config.deep_crawl_strategy.traverse(
-                        start_url=url,
-                        crawler=self.crawler,
-                        config=config
-                    ):
-                        yield result
-                finally:
-                    stack.pop()
-                    deep_crawl_ctx.set(stack)
-            else:
-                result = await original_arun(url, config=config, **kwargs)
-                yield result
-        return quantum_arun
-
-
-async def collect_results(url, crawler, config):
-    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
-        setattr(crawler, "arun", getattr(crawler, "original_arun"))
-
-    ret = crawler.arun(url, config=config)
-    # If arun is an async generator, iterate over it
-    if inspect.isasyncgen(ret):
-        return [r async for r in ret]
-    # Otherwise, await the coroutine and normalize to a list
-    result = await ret
-    return result if isinstance(result, list) else [result]
-
-async def collect_many_results(url, crawler, config):
-    # Replace back arun to its original implementation
-    if id(getattr(crawler, "arun")) != id(getattr(crawler, "original_arun")):
-        setattr(crawler, "arun", getattr(crawler, "original_arun"))
-    ret = crawler.arun_many(url, config=config)
-    # If arun is an async generator, iterate over it
-    if inspect.isasyncgen(ret):
-        return [r async for r in ret]
-    # Otherwise, await the coroutine and normalize to a list
-    result = await ret
-    return result if isinstance(result, list) else [result]
-
-
-# ------ Deep Crawl Strategy Interface ------ #
-CrawlResultT = TypeVar("CrawlResultT", bound=CrawlResult)
-# In batch mode we return List[CrawlResult] and in stream mode an AsyncGenerator.
-RunManyReturn = Union[CrawlResultT, List[CrawlResultT], AsyncGenerator[CrawlResultT, None]]
-
-
-class DeepCrawlStrategy(ABC):
-    """Abstract base class that will make Dijkstra smile"""
-    @abstractmethod
-    async def traverse(self,
-                      start_url: str,
-                      crawler: AsyncWebCrawler,
-                      config: CrawlerRunConfig) -> RunManyReturn:
-        """Traverse with O(1) memory complexity via generator fusion"""
-        ...
-
-    @abstractmethod
-    def precompute_priority(self, url: str) -> Awaitable[float]:
-        """Quantum-inspired priority precomputation"""
-        pass
-
-    @abstractmethod
-    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
-        """Hilbert-curve optimized link generation"""
-        pass
-
-# ------ BFS That Would Make Knuth Proud ------ #
-
-def calculate_quantum_batch_size(
-    depth: int,
-    max_depth: int,
-    frontier_size: int,
-    visited_size: int
-) -> int:
-    """
-    Calculates optimal batch size for URL processing using quantum-inspired mathematical principles.
-    
-    This function implements a sophisticated batch size calculation using:
-    1. Golden Ratio (φ) based scaling for optimal irrationality
-    2. Depth-aware amplitude modulation
-    3. Harmonic series dampening
-    4. Logarithmic growth control
-    5. Dynamic frontier adaptation
-    
-    The formula follows the quantum harmonic oscillator principle:
-        N = ⌈φ^(2d) * log₂(|V|) * H(d)⁻¹ * min(20, |F|/10)⌉
-    where:
-        φ = Golden Ratio ((1 + √5) / 2)
-        d = depth factor (normalized remaining depth)
-        |V| = size of visited set
-        H(d) = d-th harmonic number
-        |F| = frontier size
-    
-    Args:
-        depth (int): Current traversal depth
-        max_depth (int): Maximum allowed depth
-        frontier_size (int): Current size of frontier queue
-        visited_size (int): Number of URLs visited so far
-    
-    Returns:
-        int: Optimal batch size bounded between 1 and 100
-        
-    Mathematical Properties:
-        - Maintains O(log n) growth with respect to visited size
-        - Provides φ-optimal distribution of resources
-        - Ensures quantum-like state transitions between depths
-        - Harmonically dampened to prevent exponential explosion
-    """
-    # Golden ratio φ = (1 + √5) / 2
-    φ = (1 + 5 ** 0.5) / 2
-    
-    # Calculate normalized depth factor [0, 1]
-    depth_factor = (max_depth - depth) / max_depth if depth < max_depth else 0
-    
-    # Compute harmonic number for current depth
-    harmonic = sum(1/k for k in range(1, depth + 2))
-    
-    # Calculate quantum batch size
-    batch_size = int(np.ceil(
-        (φ ** (depth_factor * 2)) *          # Golden ratio scaling
-        np.log2(visited_size + 2) *          # Logarithmic growth factor
-        (1 / harmonic) *                     # Harmonic dampening
-        max(1, min(20, frontier_size / 10))  # Frontier-aware scaling
-    ))
-    
-    # Enforce practical bounds
-    return max(1, min(100, batch_size))
-
-
-class BFSDeepCrawlStrategy(DeepCrawlStrategy):
-    """Breadth-First Search with Einstein-Rosen bridge optimization"""
-    __slots__ = ('max_depth', 'filter_chain', 'priority_fn', 'stats', '_cancel')
-
-    def __init__(self,
-                 max_depth: int,
-                 filter_chain: FilterChain = FilterChain(),
-                 priority_fn: Callable[[str], Awaitable[float]] = lambda url: 1.0,
-                 logger: logging.Logger = None):
-        self.max_depth = max_depth
-        self.filter_chain = filter_chain
-        self.priority_fn = priority_fn
-        self.stats = TraversalStats()
-        self._cancel = asyncio.Event()
-        self.semaphore = asyncio.Semaphore(1000)
-
-    async def traverse(self,
-                      start_url: str,
-                      crawler: AsyncWebCrawler,
-                      config: CrawlerRunConfig) -> RunManyReturn:
-        """Non-blocking BFS with O(b^d) time complexity awareness"""
-        ctx = TraversalContext(self.priority_fn)
-        ctx.frontier.insert(self.priority_fn(start_url), (start_url, None, 0))
-        ctx.visited.add(start_url)
-        ctx.depths[start_url] = 0
-
-        while not ctx.frontier.is_empty() and not self._cancel.is_set():
-            # Use the best algorith, to find top_n value
-            top_n = calculate_quantum_batch_size(
-                depth=ctx.current_depth,
-                max_depth=self.max_depth,
-                frontier_size=len(ctx.frontier._heap),
-                visited_size=len(ctx.visited)
-            )
-
-            urls = ctx.frontier.extract(top_n=top_n)
-            # url, parent, depth = ctx.frontier.extract(top_n=top_n)
-            if urls:
-                ctx.current_depth = urls[0][2]
-
-            async with self.semaphore:
-                results = await collect_many_results([url for (url, parent, depth) in urls], crawler, config)
-                # results = await asyncio.gather(*[
-                #     collect_results(url, crawler, config) for (url, parent, depth) in urls
-                # ])
-                # result = _result[0]
-                for ix, result in enumerate(results):
-                    url, parent, depth = result.url, urls[ix][1], urls[ix][2]
-                    result.metadata['depth'] = depth
-                    result.metadata['parent'] = parent
-                    yield result
-
-                    if depth < self.max_depth:
-                        async for link in self.link_hypercube(result):
-                            if link not in ctx.visited:
-                                priority = self.priority_fn(link)
-                                ctx.frontier.insert(priority, (link, url, depth + 1))
-                                ctx.visited.add(link)
-                                ctx.depths[link] = depth + 1
-
-    @lru_cache(maxsize=65536)
-    async def validate_url(self, url: str) -> bool:
-        """Memoized URL validation with λ-calculus purity"""
-        try:
-            parsed = urlparse(url)
-            return (parsed.scheme in {'http', 'https'}
-                    and '.' in parsed.netloc
-                    and await self.filter_chain.apply(url))
-        except Exception:
-            return False
-
-    async def link_hypercube(self, result: CrawlResult) -> AsyncGenerator[str, None]:
-        """Hilbert-ordered link generation with O(1) yield latency"""
-        links = (link['href'] for link in result.links.get('internal', []))
-        validated = filter(self.validate_url, links)
-        for link in sorted(validated, key=lambda x: -self.priority_fn(x)):
-            yield link
-
-    def __aiter__(self) -> AsyncGenerator[CrawlResult, None]:
-        """Native async iterator interface"""
-        return self.traverse()
-
-    async def __anext__(self) -> CrawlResult:
-        """True async iterator protocol implementation"""
-        result = await self.traverse().__anext__()
-        if result:
-            return result
-        raise StopAsyncIteration
-
-    async def precompute_priority(self, url):
-        return super().precompute_priority(url)
-
-    async def shutdown(self):
-        self._cancel.set()
-
-# ------ Usage That Will Drop Jaws ------ #
-async def main():
-    """Quantum crawl example"""
-    strategy = BFSDeepCrawlStrategy(
-        max_depth=2,
-        priority_fn=lambda url: 1.0 / (len(url) + 1e-9),  # Inverse length priority
-        # filter_chain=FilterChain(...)
-    )
-
-    config: CrawlerRunConfig = CrawlerRunConfig(
-        deep_crawl_strategy=strategy,
-        stream=False,
-        verbose=True,
-        cache_mode=CacheMode.BYPASS
-    )
-
-    async with AsyncWebCrawler() as crawler:
-        run_decorator = DeepCrawlDecorator(crawler)
-        setattr(crawler, "original_arun", crawler.arun)
-        crawler.arun = run_decorator(crawler.arun)
-        start_time = time.perf_counter()
-        async for result in crawler.arun("https://docs.crawl4ai.com", config=config):
-            print(f"🌀 {result.url} (Depth: {result.metadata['depth']})")
-        print(f"Deep crawl completed in {time.perf_counter() - start_time:.2f}s")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

crawl4ai/deep_crawling/dfs_strategy.py

@@ -1,110 +0,0 @@
-# dfs_deep_crawl_strategy.py
-from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
-
-from ..models import CrawlResult
-from .bfs_strategy import BFSDeepCrawlStrategy  # noqa
-from ..types import AsyncWebCrawler, CrawlerRunConfig
-
-class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
-    """
-    Depth-First Search (DFS) deep crawling strategy.
-
-    Inherits URL validation and link discovery from BFSDeepCrawlStrategy.
-    Overrides _arun_batch and _arun_stream to use a stack (LIFO) for DFS traversal.
-    """
-    async def _arun_batch(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> List[CrawlResult]:
-        """
-        Batch (non-streaming) DFS mode.
-        Uses a stack to traverse URLs in DFS order, aggregating CrawlResults into a list.
-        """
-        visited: Set[str] = set()
-        # Stack items: (url, parent_url, depth)
-        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
-        depths: Dict[str, int] = {start_url: 0}
-        results: List[CrawlResult] = []
-
-        while stack and not self._cancel_event.is_set():
-            url, parent, depth = stack.pop()
-            if url in visited or depth > self.max_depth:
-                continue
-            visited.add(url)
-
-            # Clone config to disable recursive deep crawling.
-            batch_config = config.clone(deep_crawl_strategy=None, stream=False)
-            url_results = await crawler.arun_many(urls=[url], config=batch_config)
-            
-            for result in url_results:
-                result.metadata = result.metadata or {}
-                result.metadata["depth"] = depth
-                result.metadata["parent_url"] = parent
-                if self.url_scorer:
-                    result.metadata["score"] = self.url_scorer.score(url)
-                results.append(result)
-                
-                # Count only successful crawls toward max_pages limit
-                if result.success:
-                    self._pages_crawled += 1
-                    # Check if we've reached the limit during batch processing
-                    if self._pages_crawled >= self.max_pages:
-                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
-                        break  # Exit the generator
-                    
-                    # Only discover links from successful crawls
-                    new_links: List[Tuple[str, Optional[str]]] = []
-                    await self.link_discovery(result, url, depth, visited, new_links, depths)
-                    
-                    # Push new links in reverse order so the first discovered is processed next.
-                    for new_url, new_parent in reversed(new_links):
-                        new_depth = depths.get(new_url, depth + 1)
-                        stack.append((new_url, new_parent, new_depth))
-        return results
-
-    async def _arun_stream(
-        self,
-        start_url: str,
-        crawler: AsyncWebCrawler,
-        config: CrawlerRunConfig,
-    ) -> AsyncGenerator[CrawlResult, None]:
-        """
-        Streaming DFS mode.
-        Uses a stack to traverse URLs in DFS order and yields CrawlResults as they become available.
-        """
-        visited: Set[str] = set()
-        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
-        depths: Dict[str, int] = {start_url: 0}
-
-        while stack and not self._cancel_event.is_set():
-            url, parent, depth = stack.pop()
-            if url in visited or depth > self.max_depth:
-                continue
-            visited.add(url)
-
-            stream_config = config.clone(deep_crawl_strategy=None, stream=True)
-            stream_gen = await crawler.arun_many(urls=[url], config=stream_config)
-            async for result in stream_gen:
-                result.metadata = result.metadata or {}
-                result.metadata["depth"] = depth
-                result.metadata["parent_url"] = parent
-                if self.url_scorer:
-                    result.metadata["score"] = self.url_scorer.score(url)
-                yield result
-
-                # Only count successful crawls toward max_pages limit
-                # and only discover links from successful crawls
-                if result.success:
-                    self._pages_crawled += 1
-                    # Check if we've reached the limit during batch processing
-                    if self._pages_crawled >= self.max_pages:
-                        self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
-                        break  # Exit the generator
-                    
-                    new_links: List[Tuple[str, Optional[str]]] = []
-                    await self.link_discovery(result, url, depth, visited, new_links, depths)
-                    for new_url, new_parent in reversed(new_links):
-                        new_depth = depths.get(new_url, depth + 1)
-                        stack.append((new_url, new_parent, new_depth))

crawl4ai/deep_crawling/filters.py

@@ -1,694 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List, Pattern, Set, Union
-from urllib.parse import urlparse
-from array import array
-import re
-import logging
-from functools import lru_cache
-import fnmatch
-from dataclasses import dataclass
-import weakref
-import math
-from collections import defaultdict
-from typing import Dict
-from ..utils import HeadPeekr
-import asyncio
-import inspect
-
-
-@dataclass
-class FilterStats:
-    __slots__ = ("_counters",)
-
-    def __init__(self):
-        # Use array of unsigned ints for atomic operations
-        self._counters = array("I", [0, 0, 0])  # total, passed, rejected
-
-    @property
-    def total_urls(self):
-        return self._counters[0]
-
-    @property
-    def passed_urls(self):
-        return self._counters[1]
-
-    @property
-    def rejected_urls(self):
-        return self._counters[2]
-
-
-class URLFilter(ABC):
-    """Optimized base filter class"""
-
-    __slots__ = ("name", "stats", "_logger_ref")
-
-    def __init__(self, name: str = None):
-        self.name = name or self.__class__.__name__
-        self.stats = FilterStats()
-        # Lazy logger initialization using weakref
-        self._logger_ref = None
-
-    @property
-    def logger(self):
-        if self._logger_ref is None or self._logger_ref() is None:
-            logger = logging.getLogger(f"urlfilter.{self.name}")
-            self._logger_ref = weakref.ref(logger)
-        return self._logger_ref()
-
-    @abstractmethod
-    def apply(self, url: str) -> bool:
-        pass
-
-    def _update_stats(self, passed: bool):
-        # Use direct array index for speed
-        self.stats._counters[0] += 1  # total
-        self.stats._counters[1] += passed  # passed
-        self.stats._counters[2] += not passed  # rejected
-
-
-class FilterChain:
-    """Optimized filter chain"""
-
-    __slots__ = ("filters", "stats", "_logger_ref")
-
-    def __init__(self, filters: List[URLFilter] = None):
-        self.filters = tuple(filters or [])  # Immutable tuple for speed
-        self.stats = FilterStats()
-        self._logger_ref = None
-
-    @property
-    def logger(self):
-        if self._logger_ref is None or self._logger_ref() is None:
-            logger = logging.getLogger("urlfilter.chain")
-            self._logger_ref = weakref.ref(logger)
-        return self._logger_ref()
-
-    def add_filter(self, filter_: URLFilter) -> "FilterChain":
-        """Add a filter to the chain"""
-        self.filters.append(filter_)
-        return self  # Enable method chaining
-
-    async def apply(self, url: str) -> bool:
-        """Apply all filters concurrently when possible"""
-        self.stats._counters[0] += 1  # Total processed URLs
-
-        tasks = []
-        for f in self.filters:
-            result = f.apply(url)
-
-            if inspect.isawaitable(result):
-                tasks.append(result)  # Collect async tasks
-            elif not result:  # Sync rejection
-                self.stats._counters[2] += 1  # Sync rejected
-                return False
-
-        if tasks:
-            results = await asyncio.gather(*tasks)
-
-            # Count how many filters rejected
-            rejections = results.count(False)
-            self.stats._counters[2] += rejections
-
-            if not all(results):
-                return False  # Stop early if any filter rejected
-
-        self.stats._counters[1] += 1  # Passed
-        return True
-
-
-class URLPatternFilter(URLFilter):
-    """Pattern filter balancing speed and completeness"""
-
-    __slots__ = (
-        "patterns",  # Store original patterns for serialization
-        "use_glob",  # Store original use_glob for serialization  
-        "reverse",   # Store original reverse for serialization
-        "_simple_suffixes",
-        "_simple_prefixes",
-        "_domain_patterns",
-        "_path_patterns",
-        "_reverse",
-    )
-
-    PATTERN_TYPES = {
-        "SUFFIX": 1,  # *.html
-        "PREFIX": 2,  # /foo/*
-        "DOMAIN": 3,  # *.example.com
-        "PATH": 4,  # Everything else
-        "REGEX": 5,
-    }
-
-    def __init__(
-        self,
-        patterns: Union[str, Pattern, List[Union[str, Pattern]]],
-        use_glob: bool = True,
-        reverse: bool = False,
-    ):
-        super().__init__()
-        # Store original constructor params for serialization
-        self.patterns = patterns
-        self.use_glob = use_glob
-        self.reverse = reverse
-        
-        self._reverse = reverse
-        patterns = [patterns] if isinstance(patterns, (str, Pattern)) else patterns
-
-        self._simple_suffixes = set()
-        self._simple_prefixes = set()
-        self._domain_patterns = []
-        self._path_patterns = []
-
-        for pattern in patterns:
-            pattern_type = self._categorize_pattern(pattern)
-            self._add_pattern(pattern, pattern_type)
-
-    def _categorize_pattern(self, pattern: str) -> int:
-        """Categorize pattern for specialized handling"""
-        if not isinstance(pattern, str):
-            return self.PATTERN_TYPES["PATH"]
-
-        # Check if it's a regex pattern
-        if pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern:
-            return self.PATTERN_TYPES["REGEX"]
-
-        if pattern.count("*") == 1:
-            if pattern.startswith("*."):
-                return self.PATTERN_TYPES["SUFFIX"]
-            if pattern.endswith("/*"):
-                return self.PATTERN_TYPES["PREFIX"]
-
-        if "://" in pattern and pattern.startswith("*."):
-            return self.PATTERN_TYPES["DOMAIN"]
-
-        return self.PATTERN_TYPES["PATH"]
-
-    def _add_pattern(self, pattern: str, pattern_type: int):
-        """Add pattern to appropriate matcher"""
-        if pattern_type == self.PATTERN_TYPES["REGEX"]:
-            # For regex patterns, compile directly without glob translation
-            if isinstance(pattern, str) and (
-                pattern.startswith("^") or pattern.endswith("$") or "\\d" in pattern
-            ):
-                self._path_patterns.append(re.compile(pattern))
-                return
-        elif pattern_type == self.PATTERN_TYPES["SUFFIX"]:
-            self._simple_suffixes.add(pattern[2:])
-        elif pattern_type == self.PATTERN_TYPES["PREFIX"]:
-            self._simple_prefixes.add(pattern[:-2])
-        elif pattern_type == self.PATTERN_TYPES["DOMAIN"]:
-            self._domain_patterns.append(re.compile(pattern.replace("*.", r"[^/]+\.")))
-        else:
-            if isinstance(pattern, str):
-                # Handle complex glob patterns
-                if "**" in pattern:
-                    pattern = pattern.replace("**", ".*")
-                if "{" in pattern:
-                    # Convert {a,b} to (a|b)
-                    pattern = re.sub(
-                        r"\{([^}]+)\}",
-                        lambda m: f'({"|".join(m.group(1).split(","))})',
-                        pattern,
-                    )
-                pattern = fnmatch.translate(pattern)
-            self._path_patterns.append(
-                pattern if isinstance(pattern, Pattern) else re.compile(pattern)
-            )
-
-    @lru_cache(maxsize=10000)
-    def apply(self, url: str) -> bool:
-        # Quick suffix check (*.html)
-        if self._simple_suffixes:
-            path = url.split("?")[0]
-            if path.split("/")[-1].split(".")[-1] in self._simple_suffixes:
-                result = True
-                self._update_stats(result)
-                return not result if self._reverse else result
-
-        # Domain check
-        if self._domain_patterns:
-            for pattern in self._domain_patterns:
-                if pattern.match(url):
-                    result = True
-                    self._update_stats(result)
-                    return not result if self._reverse else result
-
-        # Prefix check (/foo/*)
-        if self._simple_prefixes:
-            path = url.split("?")[0]
-            # if any(path.startswith(p) for p in self._simple_prefixes):
-            #     result = True
-            #     self._update_stats(result)
-            #     return not result if self._reverse else result
-            ####
-            # Modified the prefix matching logic to ensure path boundary checking:
-            # - Check if the matched prefix is followed by a path separator (`/`), query parameter (`?`), fragment (`#`), or is at the end of the path
-            # - This ensures `/api/` only matches complete path segments, not substrings like `/apiv2/`
-            ####
-            for prefix in self._simple_prefixes:
-                if path.startswith(prefix):
-                    if len(path) == len(prefix) or path[len(prefix)] in ['/', '?', '#']:
-                        result = True
-                        self._update_stats(result)
-                        return not result if self._reverse else result
-
-        # Complex patterns
-        if self._path_patterns:
-            if any(p.search(url) for p in self._path_patterns):
-                result = True
-                self._update_stats(result)
-                return not result if self._reverse else result
-
-        result = False
-        self._update_stats(result)
-        return not result if self._reverse else result
-
-
-class ContentTypeFilter(URLFilter):
-    """Optimized content type filter using fast lookups"""
-
-    __slots__ = ("allowed_types", "_ext_map", "_check_extension")
-
-    # Fast extension to mime type mapping
-    _MIME_MAP = {
-        # Text Formats
-        "txt": "text/plain",
-        "html": "text/html",
-        "htm": "text/html",
-        "xhtml": "application/xhtml+xml",
-        "css": "text/css",
-        "csv": "text/csv",
-        "ics": "text/calendar",
-        "js": "application/javascript",
-        # Images
-        "bmp": "image/bmp",
-        "gif": "image/gif",
-        "jpeg": "image/jpeg",
-        "jpg": "image/jpeg",
-        "png": "image/png",
-        "svg": "image/svg+xml",
-        "tiff": "image/tiff",
-        "ico": "image/x-icon",
-        "webp": "image/webp",
-        # Audio
-        "mp3": "audio/mpeg",
-        "wav": "audio/wav",
-        "ogg": "audio/ogg",
-        "m4a": "audio/mp4",
-        "aac": "audio/aac",
-        # Video
-        "mp4": "video/mp4",
-        "mpeg": "video/mpeg",
-        "webm": "video/webm",
-        "avi": "video/x-msvideo",
-        "mov": "video/quicktime",
-        "flv": "video/x-flv",
-        "wmv": "video/x-ms-wmv",
-        "mkv": "video/x-matroska",
-        # Applications
-        "json": "application/json",
-        "xml": "application/xml",
-        "pdf": "application/pdf",
-        "zip": "application/zip",
-        "gz": "application/gzip",
-        "tar": "application/x-tar",
-        "rar": "application/vnd.rar",
-        "7z": "application/x-7z-compressed",
-        "exe": "application/vnd.microsoft.portable-executable",
-        "msi": "application/x-msdownload",
-        # Fonts
-        "woff": "font/woff",
-        "woff2": "font/woff2",
-        "ttf": "font/ttf",
-        "otf": "font/otf",
-        # Microsoft Office
-        "doc": "application/msword",
-        "dot": "application/msword",
-        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-        "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-        "xls": "application/vnd.ms-excel",
-        "ppt": "application/vnd.ms-powerpoint",
-        "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        # OpenDocument Formats
-        "odt": "application/vnd.oasis.opendocument.text",
-        "ods": "application/vnd.oasis.opendocument.spreadsheet",
-        "odp": "application/vnd.oasis.opendocument.presentation",
-        # Archives
-        "tar.gz": "application/gzip",
-        "tgz": "application/gzip",
-        "bz2": "application/x-bzip2",
-        # Others
-        "rtf": "application/rtf",
-        "apk": "application/vnd.android.package-archive",
-        "epub": "application/epub+zip",
-        "jar": "application/java-archive",
-        "swf": "application/x-shockwave-flash",
-        "midi": "audio/midi",
-        "mid": "audio/midi",
-        "ps": "application/postscript",
-        "ai": "application/postscript",
-        "eps": "application/postscript",
-        # Custom or less common
-        "bin": "application/octet-stream",
-        "dmg": "application/x-apple-diskimage",
-        "iso": "application/x-iso9660-image",
-        "deb": "application/x-debian-package",
-        "rpm": "application/x-rpm",
-        "sqlite": "application/vnd.sqlite3",
-        # Placeholder
-        "unknown": "application/octet-stream",  # Fallback for unknown file types
-        # php
-        "php": "application/x-httpd-php",
-        "php3": "application/x-httpd-php",
-        "php4": "application/x-httpd-php",
-        "php5": "application/x-httpd-php",
-        "php7": "application/x-httpd-php",
-        "phtml": "application/x-httpd-php",
-        "phps": "application/x-httpd-php-source",
-
-    }
-
-    @staticmethod
-    @lru_cache(maxsize=1000)
-    def _extract_extension(url: str) -> str:
-        """Extracts file extension from a URL."""
-        # Remove scheme (http://, https://) if present
-        if "://" in url:
-            url = url.split("://", 1)[-1]  # Get everything after '://'
-
-        # Remove domain (everything up to the first '/')
-        path_start = url.find("/")
-        path = url[path_start:] if path_start != -1 else ""
-
-        # Extract last filename in path
-        filename = path.rsplit("/", 1)[-1] if "/" in path else ""
-
-        # Extract and validate extension
-        if "." not in filename:
-            return ""
-
-        return filename.rpartition(".")[-1].lower()
-
-    def __init__(
-        self,
-        allowed_types: Union[str, List[str]],
-        check_extension: bool = True,
-        ext_map: Dict[str, str] = _MIME_MAP,
-    ):
-        super().__init__()
-        # Normalize and store as frozenset for fast lookup
-        self.allowed_types = frozenset(
-            t.lower()
-            for t in (
-                allowed_types if isinstance(allowed_types, list) else [allowed_types]
-            )
-        )
-        self._check_extension = check_extension
-
-        # Pre-compute extension map for allowed types
-        self._ext_map = frozenset(
-            ext
-            for ext, mime in self._MIME_MAP.items()
-            if any(allowed in mime for allowed in self.allowed_types)
-        )
-
-    @lru_cache(maxsize=1000)
-    def _check_url_cached(self, url: str) -> bool:
-        """Cached URL checking"""
-        if not self._check_extension:
-            return True
-        ext = self._extract_extension(url)
-        if not ext:
-            return True
-
-        return ext in self._ext_map
-
-    def apply(self, url: str) -> bool:
-        """Fast extension check with caching"""
-        result = self._check_url_cached(url)
-        self._update_stats(result)
-        return result
-
-
-class DomainFilter(URLFilter):
-    """Optimized domain filter with fast lookups and caching"""
-
-    __slots__ = ("_allowed_domains", "_blocked_domains", "_domain_cache")
-
-    # Regex for fast domain extraction
-    _DOMAIN_REGEX = re.compile(r"://([^/]+)")
-
-    def __init__(
-        self,
-        allowed_domains: Union[str, List[str]] = None,
-        blocked_domains: Union[str, List[str]] = None,
-    ):
-        super().__init__()
-
-        # Convert inputs to frozensets for immutable, fast lookups
-        self._allowed_domains = (
-            frozenset(self._normalize_domains(allowed_domains))
-            if allowed_domains
-            else None
-        )
-        self._blocked_domains = (
-            frozenset(self._normalize_domains(blocked_domains))
-            if blocked_domains
-            else frozenset()
-        )
-
-    @staticmethod
-    def _normalize_domains(domains: Union[str, List[str]]) -> Set[str]:
-        """Fast domain normalization"""
-        if isinstance(domains, str):
-            return {domains.lower()}
-        return {d.lower() for d in domains}
-    
-    @staticmethod
-    def _is_subdomain(domain: str, parent_domain: str) -> bool:
-        """Check if domain is a subdomain of parent_domain"""
-        return domain == parent_domain or domain.endswith(f".{parent_domain}")
-
-    @staticmethod
-    @lru_cache(maxsize=10000)
-    def _extract_domain(url: str) -> str:
-        """Ultra-fast domain extraction with regex and caching"""
-        match = DomainFilter._DOMAIN_REGEX.search(url)
-        return match.group(1).lower() if match else ""
-
-    def apply(self, url: str) -> bool:
-        """Optimized domain checking with early returns"""
-        # Skip processing if no filters
-        if not self._blocked_domains and self._allowed_domains is None:
-            self._update_stats(True)
-            return True
-
-        domain = self._extract_domain(url)
-
-        # Check for blocked domains, including subdomains
-        for blocked in self._blocked_domains:
-            if self._is_subdomain(domain, blocked):
-                self._update_stats(False)
-                return False
-
-        # If no allowed domains specified, accept all non-blocked
-        if self._allowed_domains is None:
-            self._update_stats(True)
-            return True
-
-        # Check if domain matches any allowed domain (including subdomains)
-        for allowed in self._allowed_domains:
-            if self._is_subdomain(domain, allowed):
-                self._update_stats(True)
-                return True
-
-        # No matches found
-        self._update_stats(False)
-        return False
-
-
-class ContentRelevanceFilter(URLFilter):
-    """BM25-based relevance filter using head section content"""
-
-    __slots__ = ("query_terms", "threshold", "k1", "b", "avgdl")
-
-    def __init__(
-        self,
-        query: str,
-        threshold: float,
-        k1: float = 1.2,
-        b: float = 0.75,
-        avgdl: int = 1000,
-    ):
-        super().__init__(name="BM25RelevanceFilter")
-        self.query_terms = self._tokenize(query)
-        self.threshold = threshold
-        self.k1 = k1  # TF saturation parameter
-        self.b = b  # Length normalization parameter
-        self.avgdl = avgdl  # Average document length (empirical value)
-
-    async def apply(self, url: str) -> bool:
-        head_content = await HeadPeekr.peek_html(url)
-        if not head_content:
-            self._update_stats(False)
-            return False
-
-        # Field extraction with weighting
-        fields = {
-            "title": HeadPeekr.get_title(head_content) or "",
-            "meta": HeadPeekr.extract_meta_tags(head_content),
-        }
-        doc_text = self._build_document(fields)
-
-        score = self._bm25(doc_text)
-        decision = score >= self.threshold
-        self._update_stats(decision)
-        return decision
-
-    def _build_document(self, fields: Dict) -> str:
-        """Weighted document construction"""
-        return " ".join(
-            [
-                fields["title"] * 3,  # Title weight
-                fields["meta"].get("description", "") * 2,
-                fields["meta"].get("keywords", ""),
-                " ".join(fields["meta"].values()),
-            ]
-        )
-
-    def _tokenize(self, text: str) -> List[str]:
-        """Fast case-insensitive tokenization"""
-        return text.lower().split()
-
-    def _bm25(self, document: str) -> float:
-        """Optimized BM25 implementation for head sections"""
-        doc_terms = self._tokenize(document)
-        doc_len = len(doc_terms)
-        tf = defaultdict(int)
-
-        for term in doc_terms:
-            tf[term] += 1
-
-        score = 0.0
-        for term in set(self.query_terms):
-            term_freq = tf[term]
-            idf = math.log((1 + 1) / (term_freq + 0.5) + 1)  # Simplified IDF
-            numerator = term_freq * (self.k1 + 1)
-            denominator = term_freq + self.k1 * (
-                1 - self.b + self.b * (doc_len / self.avgdl)
-            )
-            score += idf * (numerator / denominator)
-
-        return score
-
-
-class SEOFilter(URLFilter):
-    """Quantitative SEO quality assessment filter using head section analysis"""
-
-    __slots__ = ("threshold", "_weights", "_kw_patterns")
-
-    # Based on SEMrush/Google ranking factors research
-    DEFAULT_WEIGHTS = {
-        "title_length": 0.15,
-        "title_kw": 0.18,
-        "meta_description": 0.12,
-        "canonical": 0.10,
-        "robot_ok": 0.20,  # Most critical factor
-        "schema_org": 0.10,
-        "url_quality": 0.15,
-    }
-
-    def __init__(
-        self,
-        threshold: float = 0.65,
-        keywords: List[str] = None,
-        weights: Dict[str, float] = None,
-    ):
-        super().__init__(name="SEOFilter")
-        self.threshold = threshold
-        self._weights = weights or self.DEFAULT_WEIGHTS
-        self._kw_patterns = (
-            re.compile(
-                r"\b({})\b".format("|".join(map(re.escape, keywords or []))), re.I
-            )
-            if keywords
-            else None
-        )
-
-    async def apply(self, url: str) -> bool:
-        head_content = await HeadPeekr.peek_html(url)
-        if not head_content:
-            self._update_stats(False)
-            return False
-
-        meta = HeadPeekr.extract_meta_tags(head_content)
-        title = HeadPeekr.get_title(head_content) or ""
-        parsed_url = urlparse(url)
-
-        scores = {
-            "title_length": self._score_title_length(title),
-            "title_kw": self._score_keyword_presence(title),
-            "meta_description": self._score_meta_description(
-                meta.get("description", "")
-            ),
-            "canonical": self._score_canonical(meta.get("canonical"), url),
-            "robot_ok": 1.0 if "noindex" not in meta.get("robots", "") else 0.0,
-            "schema_org": self._score_schema_org(head_content),
-            "url_quality": self._score_url_quality(parsed_url),
-        }
-
-        total_score = sum(
-            weight * scores[factor] for factor, weight in self._weights.items()
-        )
-
-        decision = total_score >= self.threshold
-        self._update_stats(decision)
-        return decision
-
-    def _score_title_length(self, title: str) -> float:
-        length = len(title)
-        if 50 <= length <= 60:
-            return 1.0
-        if 40 <= length < 50 or 60 < length <= 70:
-            return 0.7
-        return 0.3  # Poor length
-
-    def _score_keyword_presence(self, text: str) -> float:
-        if not self._kw_patterns:
-            return 0.0
-        matches = len(self._kw_patterns.findall(text))
-        return min(matches * 0.3, 1.0)  # Max 3 matches
-
-    def _score_meta_description(self, desc: str) -> float:
-        length = len(desc)
-        if 140 <= length <= 160:
-            return 1.0
-        return 0.5 if 120 <= length <= 200 else 0.2
-
-    def _score_canonical(self, canonical: str, original: str) -> float:
-        if not canonical:
-            return 0.5  # Neutral score
-        return 1.0 if canonical == original else 0.2
-
-    def _score_schema_org(self, html: str) -> float:
-        # Detect any schema.org markup in head
-        return (
-            1.0
-            if re.search(r'<script[^>]+type=["\']application/ld\+json', html)
-            else 0.0
-        )
-
-    def _score_url_quality(self, parsed_url) -> float:
-        score = 1.0
-        path = parsed_url.path.lower()
-
-        # Penalty factors
-        if len(path) > 80:
-            score *= 0.7
-        if re.search(r"\d{4}", path):
-            score *= 0.8  # Numbers in path
-        if parsed_url.query:
-            score *= 0.6  # URL parameters
-        if "_" in path:
-            score *= 0.9  # Underscores vs hyphens
-
-        return score

crawl4ai/deep_crawling/scorers.py

@@ -1,519 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import List, Dict, Optional
-from dataclasses import dataclass
-from urllib.parse import urlparse, unquote
-import re
-import logging
-from functools import lru_cache
-from array import array
-import ctypes
-import platform
-PLATFORM = platform.system()
-
-# Pre-computed scores for common year differences
-_SCORE_LOOKUP = [1.0, 0.5, 0.3333333333333333, 0.25]
-
-# Pre-computed scores for common year differences
-_FRESHNESS_SCORES = [
-   1.0,    # Current year
-   0.9,    # Last year
-   0.8,    # 2 years ago
-   0.7,    # 3 years ago
-   0.6,    # 4 years ago
-   0.5,    # 5 years ago
-]
-
-class ScoringStats:
-    __slots__ = ('_urls_scored', '_total_score', '_min_score', '_max_score')
-    
-    def __init__(self):
-        self._urls_scored = 0
-        self._total_score = 0.0
-        self._min_score = None  # Lazy initialization
-        self._max_score = None
-    
-    def update(self, score: float) -> None:
-        """Optimized update with minimal operations"""
-        self._urls_scored += 1
-        self._total_score += score
-        
-        # Lazy min/max tracking - only if actually accessed
-        if self._min_score is not None:
-            if score < self._min_score:
-                self._min_score = score
-        if self._max_score is not None:
-            if score > self._max_score:
-                self._max_score = score
-                
-    def get_average(self) -> float:
-        """Direct calculation instead of property"""
-        return self._total_score / self._urls_scored if self._urls_scored else 0.0
-    
-    def get_min(self) -> float:
-        """Lazy min calculation"""
-        if self._min_score is None:
-            self._min_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
-        return self._min_score
-        
-    def get_max(self) -> float:
-        """Lazy max calculation"""
-        if self._max_score is None:
-            self._max_score = self._total_score / self._urls_scored if self._urls_scored else 0.0
-        return self._max_score
-class URLScorer(ABC):
-    __slots__ = ('_weight', '_stats')
-    
-    def __init__(self, weight: float = 1.0):
-        # Store weight directly as float32 for memory efficiency
-        self._weight = ctypes.c_float(weight).value
-        self._stats = ScoringStats()
-    
-    @abstractmethod
-    def _calculate_score(self, url: str) -> float:
-        """Calculate raw score for URL."""
-        pass
-    
-    def score(self, url: str) -> float:
-        """Calculate weighted score with minimal overhead."""
-        score = self._calculate_score(url) * self._weight
-        self._stats.update(score)
-        return score
-    
-    @property
-    def stats(self):
-        """Access to scoring statistics."""
-        return self._stats
-    
-    @property
-    def weight(self):
-        return self._weight
-
-class CompositeScorer(URLScorer):
-    __slots__ = ('_scorers', '_normalize', '_weights_array', '_score_array')
-    
-    def __init__(self, scorers: List[URLScorer], normalize: bool = True):
-        """Initialize composite scorer combining multiple scoring strategies.
-        
-        Optimized for:
-        - Fast parallel scoring
-        - Memory efficient score aggregation
-        - Quick short-circuit conditions
-        - Pre-allocated arrays
-        
-        Args:
-            scorers: List of scoring strategies to combine
-            normalize: Whether to normalize final score by scorer count
-        """
-        super().__init__(weight=1.0)
-        self._scorers = scorers
-        self._normalize = normalize
-        
-        # Pre-allocate arrays for scores and weights
-        self._weights_array = array('f', [s.weight for s in scorers])
-        self._score_array = array('f', [0.0] * len(scorers))
-
-    @lru_cache(maxsize=10000)
-    def _calculate_score(self, url: str) -> float:
-        """Calculate combined score from all scoring strategies.
-        
-        Uses:
-        1. Pre-allocated arrays for scores
-        2. Short-circuit on zero scores
-        3. Optimized normalization
-        4. Vectorized operations where possible
-        
-        Args:
-            url: URL to score
-            
-        Returns:
-            Combined and optionally normalized score
-        """
-        total_score = 0.0
-        scores = self._score_array
-        
-        # Get scores from all scorers
-        for i, scorer in enumerate(self._scorers):
-            # Use public score() method which applies weight
-            scores[i] = scorer.score(url)
-            total_score += scores[i]
-            
-        # Normalize if requested
-        if self._normalize and self._scorers:
-            count = len(self._scorers)
-            return total_score / count
-            
-        return total_score
-
-    def score(self, url: str) -> float:
-        """Public scoring interface with stats tracking.
-        
-        Args:
-            url: URL to score
-            
-        Returns:
-            Final combined score
-        """
-        score = self._calculate_score(url)
-        self.stats.update(score)
-        return score
-
-class KeywordRelevanceScorer(URLScorer):
-    __slots__ = ('_weight', '_stats', '_keywords', '_case_sensitive')
-    
-    def __init__(self, keywords: List[str], weight: float = 1.0, case_sensitive: bool = False):
-        super().__init__(weight=weight)
-        self._case_sensitive = case_sensitive
-        # Pre-process keywords once
-        self._keywords = [k if case_sensitive else k.lower() for k in keywords]
-    
-    @lru_cache(maxsize=10000)
-    def _url_bytes(self, url: str) -> bytes:
-        """Cache decoded URL bytes"""
-        return url.encode('utf-8') if self._case_sensitive else url.lower().encode('utf-8')
-    
-    
-    def _calculate_score(self, url: str) -> float:
-        """Fast string matching without regex or byte conversion"""
-        if not self._case_sensitive:
-            url = url.lower()
-            
-        matches = sum(1 for k in self._keywords if k in url)
-        
-        # Fast return paths
-        if not matches:
-            return 0.0
-        if matches == len(self._keywords):
-            return 1.0
-            
-        return matches / len(self._keywords)
-
-class PathDepthScorer(URLScorer):
-    __slots__ = ('_weight', '_stats', '_optimal_depth')  # Remove _url_cache
-    
-    def __init__(self, optimal_depth: int = 3, weight: float = 1.0):
-        super().__init__(weight=weight)
-        self._optimal_depth = optimal_depth
-
-    @staticmethod
-    @lru_cache(maxsize=10000)
-    def _quick_depth(path: str) -> int:
-        """Ultra fast path depth calculation.
-        
-        Examples:
-            - "http://example.com" -> 0  # No path segments
-            - "http://example.com/" -> 0  # Empty path
-            - "http://example.com/a" -> 1
-            - "http://example.com/a/b" -> 2
-        """
-        if not path or path == '/':
-            return 0
-            
-        if '/' not in path:
-            return 0
-            
-        depth = 0
-        last_was_slash = True
-        
-        for c in path:
-            if c == '/':
-                if not last_was_slash:
-                    depth += 1
-                last_was_slash = True
-            else:
-                last_was_slash = False
-                
-        if not last_was_slash:
-            depth += 1
-            
-        return depth
-
-    @lru_cache(maxsize=10000)  # Cache the whole calculation
-    def _calculate_score(self, url: str) -> float:
-        pos = url.find('/', url.find('://') + 3)
-        if pos == -1:
-            depth = 0
-        else:
-            depth = self._quick_depth(url[pos:])
-            
-        # Use lookup table for common distances
-        distance = depth - self._optimal_depth
-        distance = distance if distance >= 0 else -distance  # Faster than abs()
-        
-        if distance < 4:
-            return _SCORE_LOOKUP[distance]
-            
-        return 1.0 / (1.0 + distance)                                             
-
-class ContentTypeScorer(URLScorer):
-    __slots__ = ('_weight', '_exact_types', '_regex_types')
-
-    def __init__(self, type_weights: Dict[str, float], weight: float = 1.0):
-        """Initialize scorer with type weights map.
-        
-        Args:
-            type_weights: Dict mapping file extensions/patterns to scores (e.g. {'.html$': 1.0})
-            weight: Overall weight multiplier for this scorer
-        """
-        super().__init__(weight=weight)
-        self._exact_types = {}  # Fast lookup for simple extensions
-        self._regex_types = []  # Fallback for complex patterns
-        
-        # Split into exact vs regex matchers for performance
-        for pattern, score in type_weights.items():
-            if pattern.startswith('.') and pattern.endswith('$'):
-                ext = pattern[1:-1]
-                self._exact_types[ext] = score
-            else:
-                self._regex_types.append((re.compile(pattern), score))
-                
-        # Sort complex patterns by score for early exit
-        self._regex_types.sort(key=lambda x: -x[1])
-
-    @staticmethod
-    @lru_cache(maxsize=10000)
-    def _quick_extension(url: str) -> str:
-        """Extract file extension ultra-fast without regex/splits.
-        
-        Handles:
-        - Basic extensions: "example.html" -> "html"
-        - Query strings: "page.php?id=1" -> "php" 
-        - Fragments: "doc.pdf#page=1" -> "pdf"
-        - Path params: "file.jpg;width=100" -> "jpg"
-        
-        Args:
-            url: URL to extract extension from
-            
-        Returns:
-            Extension without dot, or empty string if none found
-        """
-        pos = url.rfind('.')
-        if pos == -1:
-            return ''
-        
-        # Find first non-alphanumeric char after extension
-        end = len(url)
-        for i in range(pos + 1, len(url)):
-            c = url[i]
-            # Stop at query string, fragment, path param or any non-alphanumeric
-            if c in '?#;' or not c.isalnum():
-                end = i
-                break
-                
-        return url[pos + 1:end].lower()
-
-    @lru_cache(maxsize=10000)
-    def _calculate_score(self, url: str) -> float:
-        """Calculate content type score for URL.
-        
-        Uses staged approach:
-        1. Try exact extension match (fast path)
-        2. Fall back to regex patterns if needed
-        
-        Args:
-            url: URL to score
-            
-        Returns:
-            Score between 0.0 and 1.0 * weight
-        """
-        # Fast path: direct extension lookup
-        ext = self._quick_extension(url)
-        if ext:
-            score = self._exact_types.get(ext, None)
-            if score is not None:
-                return score
-                
-        # Slow path: regex patterns
-        for pattern, score in self._regex_types:
-            if pattern.search(url):
-                return score
-
-        return 0.0
-
-class FreshnessScorer(URLScorer):
-    __slots__ = ('_weight', '_date_pattern', '_current_year')
-
-    def __init__(self, weight: float = 1.0, current_year: int = 2024):
-        """Initialize freshness scorer.
-        
-        Extracts and scores dates from URLs using format:
-        - YYYY/MM/DD 
-        - YYYY-MM-DD
-        - YYYY_MM_DD
-        - YYYY (year only)
-        
-        Args:
-            weight: Score multiplier
-            current_year: Year to calculate freshness against (default 2024)
-        """
-        super().__init__(weight=weight)
-        self._current_year = current_year
-        
-        # Combined pattern for all date formats
-        # Uses non-capturing groups (?:) and alternation
-        self._date_pattern = re.compile(
-            r'(?:/'  # Path separator
-            r'|[-_])'  # or date separators
-            r'((?:19|20)\d{2})'  # Year group (1900-2099)
-            r'(?:'  # Optional month/day group
-            r'(?:/|[-_])'  # Date separator  
-            r'(?:\d{2})'  # Month
-            r'(?:'  # Optional day
-            r'(?:/|[-_])'  # Date separator
-            r'(?:\d{2})'  # Day
-            r')?'  # Day is optional
-            r')?'  # Month/day group is optional
-        )
-
-    @lru_cache(maxsize=10000)
-    def _extract_year(self, url: str) -> Optional[int]:
-        """Extract the most recent year from URL.
-        
-        Args:
-            url: URL to extract year from
-            
-        Returns:
-            Year as int or None if no valid year found
-        """
-        matches = self._date_pattern.finditer(url)
-        latest_year = None
-        
-        # Find most recent year
-        for match in matches:
-            year = int(match.group(1))
-            if (year <= self._current_year and  # Sanity check
-                (latest_year is None or year > latest_year)):
-                latest_year = year
-                
-        return latest_year
-
-    @lru_cache(maxsize=10000) 
-    def _calculate_score(self, url: str) -> float:
-        """Calculate freshness score based on URL date.
-        
-        More recent years score higher. Uses pre-computed scoring
-        table for common year differences.
-        
-        Args:
-            url: URL to score
-            
-        Returns:
-            Score between 0.0 and 1.0 * weight
-        """
-        year = self._extract_year(url)
-        if year is None:
-            return 0.5  # Default score
-            
-        # Use lookup table for common year differences
-        year_diff = self._current_year - year
-        if year_diff < len(_FRESHNESS_SCORES):
-            return _FRESHNESS_SCORES[year_diff]
-            
-        # Fallback calculation for older content
-        return max(0.1, 1.0 - year_diff * 0.1)
-
-class DomainAuthorityScorer(URLScorer):
-    __slots__ = ('_weight', '_domain_weights', '_default_weight', '_top_domains')
-    
-    def __init__(
-        self,
-        domain_weights: Dict[str, float],
-        default_weight: float = 0.5,
-        weight: float = 1.0,
-    ):
-        """Initialize domain authority scorer.
-        
-        Args:
-            domain_weights: Dict mapping domains to authority scores
-            default_weight: Score for unknown domains
-            weight: Overall scorer weight multiplier
-            
-        Example:
-            {
-                'python.org': 1.0,
-                'github.com': 0.9,
-                'medium.com': 0.7
-            }
-        """
-        super().__init__(weight=weight)
-        
-        # Pre-process domains for faster lookup
-        self._domain_weights = {
-            domain.lower(): score 
-            for domain, score in domain_weights.items()
-        }
-        self._default_weight = default_weight
-        
-        # Cache top domains for fast path
-        self._top_domains = {
-            domain: score
-            for domain, score in sorted(
-                domain_weights.items(), 
-                key=lambda x: -x[1]
-            )[:5]  # Keep top 5 highest scoring domains
-        }
-
-    @staticmethod
-    @lru_cache(maxsize=10000)
-    def _extract_domain(url: str) -> str:
-        """Extract domain from URL ultra-fast.
-        
-        Handles:
-        - Basic domains: "example.com"
-        - Subdomains: "sub.example.com" 
-        - Ports: "example.com:8080"
-        - IPv4: "192.168.1.1"
-        
-        Args:
-            url: Full URL to extract domain from
-            
-        Returns:
-            Lowercase domain without port
-        """
-        # Find domain start
-        start = url.find('://') 
-        if start == -1:
-            start = 0
-        else:
-            start += 3
-            
-        # Find domain end
-        end = url.find('/', start)
-        if end == -1:
-            end = url.find('?', start)
-            if end == -1:
-                end = url.find('#', start)
-                if end == -1:
-                    end = len(url)
-                    
-        # Extract domain and remove port
-        domain = url[start:end]
-        port_idx = domain.rfind(':')
-        if port_idx != -1:
-            domain = domain[:port_idx]
-            
-        return domain.lower()
-
-    @lru_cache(maxsize=10000)
-    def _calculate_score(self, url: str) -> float:
-        """Calculate domain authority score.
-        
-        Uses staged approach:
-        1. Check top domains (fastest)
-        2. Check full domain weights
-        3. Return default weight
-        
-        Args:
-            url: URL to score
-            
-        Returns:
-            Authority score between 0.0 and 1.0 * weight
-        """
-        domain = self._extract_domain(url)
-        
-        # Fast path: check top domains first
-        score = self._top_domains.get(domain)
-        if score is not None:
-            return score
-            
-        # Regular path: check all domains
-        return self._domain_weights.get(domain, self._default_weight)

crawl4ai/docker_client.py

@@ -1,168 +0,0 @@
-from typing import List, Optional, Union, AsyncGenerator, Dict, Any
-import httpx
-import json
-from urllib.parse import urljoin
-import asyncio
-
-from .async_configs import BrowserConfig, CrawlerRunConfig
-from .models import CrawlResult
-from .async_logger import AsyncLogger, LogLevel
-
-
-class Crawl4aiClientError(Exception):
-    """Base exception for Crawl4ai Docker client errors."""
-    pass
-
-
-class ConnectionError(Crawl4aiClientError):
-    """Raised when connection to the Docker server fails."""
-    pass
-
-
-class RequestError(Crawl4aiClientError):
-    """Raised when the server returns an error response."""
-    pass
-
-
-class Crawl4aiDockerClient:
-    """Client for interacting with Crawl4AI Docker server with token authentication."""
-    
-    def __init__(
-        self,
-        base_url: str = "http://localhost:8000",
-        timeout: float = 30.0,
-        verify_ssl: bool = True,
-        verbose: bool = True,
-        log_file: Optional[str] = None
-    ):
-        self.base_url = base_url.rstrip('/')
-        self.timeout = timeout
-        self.logger = AsyncLogger(log_file=log_file, log_level=LogLevel.DEBUG, verbose=verbose)
-        self._http_client = httpx.AsyncClient(
-            timeout=timeout,
-            verify=verify_ssl,
-            headers={"Content-Type": "application/json"}
-        )
-        self._token: Optional[str] = None
-
-    async def authenticate(self, email: str) -> None:
-        """Authenticate with the server and store the token."""
-        url = urljoin(self.base_url, "/token")
-        try:
-            self.logger.info(f"Authenticating with email: {email}", tag="AUTH")
-            response = await self._http_client.post(url, json={"email": email})
-            response.raise_for_status()
-            data = response.json()
-            self._token = data["access_token"]
-            self._http_client.headers["Authorization"] = f"Bearer {self._token}"
-            self.logger.success("Authentication successful", tag="AUTH")
-        except (httpx.RequestError, httpx.HTTPStatusError) as e:
-            error_msg = f"Authentication failed: {str(e)}"
-            self.logger.error(error_msg, tag="ERROR")
-            raise ConnectionError(error_msg)
-
-    async def _check_server(self) -> None:
-        """Check if server is reachable, raising an error if not."""
-        try:
-            await self._http_client.get(urljoin(self.base_url, "/health"))
-            self.logger.success(f"Connected to {self.base_url}", tag="READY")
-        except httpx.RequestError as e:
-            self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR")
-            raise ConnectionError(f"Cannot connect to server: {str(e)}")
-
-    def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, 
-                       crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]:
-        """Prepare request data from configs."""
-        if self._token:
-            self._http_client.headers["Authorization"] = f"Bearer {self._token}"
-        return {
-            "urls": urls,
-            "browser_config": browser_config.dump() if browser_config else {},
-            "crawler_config": crawler_config.dump() if crawler_config else {}
-        }
-
-    async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
-        """Make an HTTP request with error handling."""
-        url = urljoin(self.base_url, endpoint)
-        try:
-            response = await self._http_client.request(method, url, **kwargs)
-            response.raise_for_status()
-            return response
-        except httpx.TimeoutException as e:
-            raise ConnectionError(f"Request timed out: {str(e)}")
-        except httpx.RequestError as e:
-            raise ConnectionError(f"Failed to connect: {str(e)}")
-        except httpx.HTTPStatusError as e:
-            error_msg = (e.response.json().get("detail", str(e)) 
-                        if "application/json" in e.response.headers.get("content-type", "") 
-                        else str(e))
-            raise RequestError(f"Server error {e.response.status_code}: {error_msg}")
-
-    async def crawl(
-        self,
-        urls: List[str],
-        browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None
-    ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
-        """Execute a crawl operation."""
-        await self._check_server()
-        
-        data = self._prepare_request(urls, browser_config, crawler_config)
-        is_streaming = crawler_config and crawler_config.stream
-        
-        self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL")
-        
-        if is_streaming:
-            async def stream_results() -> AsyncGenerator[CrawlResult, None]:
-                async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response:
-                    response.raise_for_status()
-                    async for line in response.aiter_lines():
-                        if line.strip():
-                            result = json.loads(line)
-                            if "error" in result:
-                                self.logger.error_status(url=result.get("url", "unknown"), error=result["error"])
-                                continue
-                            self.logger.url_status(url=result.get("url", "unknown"), success=True, timing=result.get("timing", 0.0))
-                            if result.get("status") == "completed":
-                                continue
-                            else:
-                                yield CrawlResult(**result)
-            return stream_results()
-        
-        response = await self._request("POST", "/crawl", json=data)
-        result_data = response.json()
-        if not result_data.get("success", False):
-            raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}")
-        
-        results = [CrawlResult(**r) for r in result_data.get("results", [])]
-        self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL")
-        return results[0] if len(results) == 1 else results
-
-    async def get_schema(self) -> Dict[str, Any]:
-        """Retrieve configuration schemas."""
-        response = await self._request("GET", "/schema")
-        return response.json()
-
-    async def close(self) -> None:
-        """Close the HTTP client session."""
-        self.logger.info("Closing client", tag="CLOSE")
-        await self._http_client.aclose()
-
-    async def __aenter__(self) -> "Crawl4aiDockerClient":
-        return self
-
-    async def __aexit__(self, exc_type: Optional[type], exc_val: Optional[Exception], exc_tb: Optional[Any]) -> None:
-        await self.close()
-
-
-# Example usage
-async def main():
-    async with Crawl4aiDockerClient(verbose=True) as client:
-        await client.authenticate("user@example.com")
-        result = await client.crawl(["https://example.com"])
-        print(result)
-        schema = await client.get_schema()
-        print(schema)
-
-if __name__ == "__main__":
-    asyncio.run(main())

crawl4ai/html2text/__main__.py

@@ -1,3 +0,0 @@
-from .cli import main
-
-main()

crawl4ai/html2text/_typing.py

@@ -1,3 +0,0 @@
-class OutCallback:
-    def __call__(self, s: str) -> None:
-        ...

crawl4ai/html2text/cli.py

@@ -1,330 +0,0 @@
-import argparse
-import sys
-
-from . import HTML2Text, __version__, config
-
-
-def main() -> None:
-    baseurl = ""
-
-    class bcolors:
-        HEADER = "\033[95m"
-        OKBLUE = "\033[94m"
-        OKGREEN = "\033[92m"
-        WARNING = "\033[93m"
-        FAIL = "\033[91m"
-        ENDC = "\033[0m"
-        BOLD = "\033[1m"
-        UNDERLINE = "\033[4m"
-
-    p = argparse.ArgumentParser()
-    p.add_argument(
-        "--default-image-alt",
-        dest="default_image_alt",
-        default=config.DEFAULT_IMAGE_ALT,
-        help="The default alt string for images with missing ones",
-    )
-    p.add_argument(
-        "--pad-tables",
-        dest="pad_tables",
-        action="store_true",
-        default=config.PAD_TABLES,
-        help="pad the cells to equal column width in tables",
-    )
-    p.add_argument(
-        "--no-wrap-links",
-        dest="wrap_links",
-        action="store_false",
-        default=config.WRAP_LINKS,
-        help="don't wrap links during conversion",
-    )
-    p.add_argument(
-        "--wrap-list-items",
-        dest="wrap_list_items",
-        action="store_true",
-        default=config.WRAP_LIST_ITEMS,
-        help="wrap list items during conversion",
-    )
-    p.add_argument(
-        "--wrap-tables",
-        dest="wrap_tables",
-        action="store_true",
-        default=config.WRAP_TABLES,
-        help="wrap tables",
-    )
-    p.add_argument(
-        "--ignore-emphasis",
-        dest="ignore_emphasis",
-        action="store_true",
-        default=config.IGNORE_EMPHASIS,
-        help="don't include any formatting for emphasis",
-    )
-    p.add_argument(
-        "--reference-links",
-        dest="inline_links",
-        action="store_false",
-        default=config.INLINE_LINKS,
-        help="use reference style links instead of inline links",
-    )
-    p.add_argument(
-        "--ignore-links",
-        dest="ignore_links",
-        action="store_true",
-        default=config.IGNORE_ANCHORS,
-        help="don't include any formatting for links",
-    )
-    p.add_argument(
-        "--ignore-mailto-links",
-        action="store_true",
-        dest="ignore_mailto_links",
-        default=config.IGNORE_MAILTO_LINKS,
-        help="don't include mailto: links",
-    )
-    p.add_argument(
-        "--protect-links",
-        dest="protect_links",
-        action="store_true",
-        default=config.PROTECT_LINKS,
-        help="protect links from line breaks surrounding them with angle brackets",
-    )
-    p.add_argument(
-        "--ignore-images",
-        dest="ignore_images",
-        action="store_true",
-        default=config.IGNORE_IMAGES,
-        help="don't include any formatting for images",
-    )
-    p.add_argument(
-        "--images-as-html",
-        dest="images_as_html",
-        action="store_true",
-        default=config.IMAGES_AS_HTML,
-        help=(
-            "Always write image tags as raw html; preserves `height`, `width` and "
-            "`alt` if possible."
-        ),
-    )
-    p.add_argument(
-        "--images-to-alt",
-        dest="images_to_alt",
-        action="store_true",
-        default=config.IMAGES_TO_ALT,
-        help="Discard image data, only keep alt text",
-    )
-    p.add_argument(
-        "--images-with-size",
-        dest="images_with_size",
-        action="store_true",
-        default=config.IMAGES_WITH_SIZE,
-        help=(
-            "Write image tags with height and width attrs as raw html to retain "
-            "dimensions"
-        ),
-    )
-    p.add_argument(
-        "-g",
-        "--google-doc",
-        action="store_true",
-        dest="google_doc",
-        default=False,
-        help="convert an html-exported Google Document",
-    )
-    p.add_argument(
-        "-d",
-        "--dash-unordered-list",
-        action="store_true",
-        dest="ul_style_dash",
-        default=False,
-        help="use a dash rather than a star for unordered list items",
-    )
-    p.add_argument(
-        "-e",
-        "--asterisk-emphasis",
-        action="store_true",
-        dest="em_style_asterisk",
-        default=False,
-        help="use an asterisk rather than an underscore for emphasized text",
-    )
-    p.add_argument(
-        "-b",
-        "--body-width",
-        dest="body_width",
-        type=int,
-        default=config.BODY_WIDTH,
-        help="number of characters per output line, 0 for no wrap",
-    )
-    p.add_argument(
-        "-i",
-        "--google-list-indent",
-        dest="list_indent",
-        type=int,
-        default=config.GOOGLE_LIST_INDENT,
-        help="number of pixels Google indents nested lists",
-    )
-    p.add_argument(
-        "-s",
-        "--hide-strikethrough",
-        action="store_true",
-        dest="hide_strikethrough",
-        default=False,
-        help="hide strike-through text. only relevant when -g is " "specified as well",
-    )
-    p.add_argument(
-        "--escape-all",
-        action="store_true",
-        dest="escape_snob",
-        default=False,
-        help=(
-            "Escape all special characters.  Output is less readable, but avoids "
-            "corner case formatting issues."
-        ),
-    )
-    p.add_argument(
-        "--bypass-tables",
-        action="store_true",
-        dest="bypass_tables",
-        default=config.BYPASS_TABLES,
-        help="Format tables in HTML rather than Markdown syntax.",
-    )
-    p.add_argument(
-        "--ignore-tables",
-        action="store_true",
-        dest="ignore_tables",
-        default=config.IGNORE_TABLES,
-        help="Ignore table-related tags (table, th, td, tr) " "while keeping rows.",
-    )
-    p.add_argument(
-        "--single-line-break",
-        action="store_true",
-        dest="single_line_break",
-        default=config.SINGLE_LINE_BREAK,
-        help=(
-            "Use a single line break after a block element rather than two line "
-            "breaks. NOTE: Requires --body-width=0"
-        ),
-    )
-    p.add_argument(
-        "--unicode-snob",
-        action="store_true",
-        dest="unicode_snob",
-        default=config.UNICODE_SNOB,
-        help="Use unicode throughout document",
-    )
-    p.add_argument(
-        "--no-automatic-links",
-        action="store_false",
-        dest="use_automatic_links",
-        default=config.USE_AUTOMATIC_LINKS,
-        help="Do not use automatic links wherever applicable",
-    )
-    p.add_argument(
-        "--no-skip-internal-links",
-        action="store_false",
-        dest="skip_internal_links",
-        default=config.SKIP_INTERNAL_LINKS,
-        help="Do not skip internal links",
-    )
-    p.add_argument(
-        "--links-after-para",
-        action="store_true",
-        dest="links_each_paragraph",
-        default=config.LINKS_EACH_PARAGRAPH,
-        help="Put links after each paragraph instead of document",
-    )
-    p.add_argument(
-        "--mark-code",
-        action="store_true",
-        dest="mark_code",
-        default=config.MARK_CODE,
-        help="Mark program code blocks with [code]...[/code]",
-    )
-    p.add_argument(
-        "--decode-errors",
-        dest="decode_errors",
-        default=config.DECODE_ERRORS,
-        help=(
-            "What to do in case of decode errors.'ignore', 'strict' and 'replace' are "
-            "acceptable values"
-        ),
-    )
-    p.add_argument(
-        "--open-quote",
-        dest="open_quote",
-        default=config.OPEN_QUOTE,
-        help="The character used to open quotes",
-    )
-    p.add_argument(
-        "--close-quote",
-        dest="close_quote",
-        default=config.CLOSE_QUOTE,
-        help="The character used to close quotes",
-    )
-    p.add_argument(
-        "--version", action="version", version=".".join(map(str, __version__))
-    )
-    p.add_argument("filename", nargs="?")
-    p.add_argument("encoding", nargs="?", default="utf-8")
-    p.add_argument(
-        "--include-sup-sub",
-        dest="include_sup_sub",
-        action="store_true",
-        default=config.INCLUDE_SUP_SUB,
-        help="Include the sup and sub tags",
-    )
-    args = p.parse_args()
-
-    if args.filename and args.filename != "-":
-        with open(args.filename, "rb") as fp:
-            data = fp.read()
-    else:
-        data = sys.stdin.buffer.read()
-
-    try:
-        html = data.decode(args.encoding, args.decode_errors)
-    except UnicodeDecodeError as err:
-        warning = bcolors.WARNING + "Warning:" + bcolors.ENDC
-        warning += " Use the " + bcolors.OKGREEN
-        warning += "--decode-errors=ignore" + bcolors.ENDC + " flag."
-        print(warning)
-        raise err
-
-    h = HTML2Text(baseurl=baseurl)
-    # handle options
-    if args.ul_style_dash:
-        h.ul_item_mark = "-"
-    if args.em_style_asterisk:
-        h.emphasis_mark = "*"
-        h.strong_mark = "__"
-
-    h.body_width = args.body_width
-    h.google_list_indent = args.list_indent
-    h.ignore_emphasis = args.ignore_emphasis
-    h.ignore_links = args.ignore_links
-    h.ignore_mailto_links = args.ignore_mailto_links
-    h.protect_links = args.protect_links
-    h.ignore_images = args.ignore_images
-    h.images_as_html = args.images_as_html
-    h.images_to_alt = args.images_to_alt
-    h.images_with_size = args.images_with_size
-    h.google_doc = args.google_doc
-    h.hide_strikethrough = args.hide_strikethrough
-    h.escape_snob = args.escape_snob
-    h.bypass_tables = args.bypass_tables
-    h.ignore_tables = args.ignore_tables
-    h.single_line_break = args.single_line_break
-    h.inline_links = args.inline_links
-    h.unicode_snob = args.unicode_snob
-    h.use_automatic_links = args.use_automatic_links
-    h.skip_internal_links = args.skip_internal_links
-    h.links_each_paragraph = args.links_each_paragraph
-    h.mark_code = args.mark_code
-    h.wrap_links = args.wrap_links
-    h.wrap_list_items = args.wrap_list_items
-    h.wrap_tables = args.wrap_tables
-    h.pad_tables = args.pad_tables
-    h.default_image_alt = args.default_image_alt
-    h.open_quote = args.open_quote
-    h.close_quote = args.close_quote
-    h.include_sup_sub = args.include_sup_sub
-
-    sys.stdout.write(h.handle(html))

crawl4ai/html2text/config.py

@@ -1,172 +0,0 @@
-import re
-
-# Use Unicode characters instead of their ascii pseudo-replacements
-UNICODE_SNOB = False
-
-# Marker to use for marking tables for padding post processing
-TABLE_MARKER_FOR_PAD = "special_marker_for_table_padding"
-# Escape all special characters.  Output is less readable, but avoids
-# corner case formatting issues.
-ESCAPE_SNOB = False
-ESCAPE_BACKSLASH = False
-ESCAPE_DOT = False
-ESCAPE_PLUS = False
-ESCAPE_DASH = False
-
-# Put the links after each paragraph instead of at the end.
-LINKS_EACH_PARAGRAPH = False
-
-# Wrap long lines at position. 0 for no wrapping.
-BODY_WIDTH = 78
-
-# Don't show internal links (href="#local-anchor") -- corresponding link
-# targets won't be visible in the plain text file anyway.
-SKIP_INTERNAL_LINKS = True
-
-# Use inline, rather than reference, formatting for images and links
-INLINE_LINKS = True
-
-# Protect links from line breaks surrounding them with angle brackets (in
-# addition to their square brackets)
-PROTECT_LINKS = False
-# WRAP_LINKS = True
-WRAP_LINKS = True
-
-# Wrap list items.
-WRAP_LIST_ITEMS = False
-
-# Wrap tables
-WRAP_TABLES = False
-
-# Number of pixels Google indents nested lists
-GOOGLE_LIST_INDENT = 36
-
-# Values Google and others may use to indicate bold text
-BOLD_TEXT_STYLE_VALUES = ("bold", "700", "800", "900")
-
-IGNORE_ANCHORS = False
-IGNORE_MAILTO_LINKS = False
-IGNORE_IMAGES = False
-IMAGES_AS_HTML = False
-IMAGES_TO_ALT = False
-IMAGES_WITH_SIZE = False
-IGNORE_EMPHASIS = False
-MARK_CODE = False
-DECODE_ERRORS = "strict"
-DEFAULT_IMAGE_ALT = ""
-PAD_TABLES = False
-
-# Convert links with same href and text to <href> format
-# if they are absolute links
-USE_AUTOMATIC_LINKS = True
-
-# For checking space-only lines on line 771
-RE_SPACE = re.compile(r"\s\+")
-
-RE_ORDERED_LIST_MATCHER = re.compile(r"\d+\.\s")
-RE_UNORDERED_LIST_MATCHER = re.compile(r"[-\*\+]\s")
-RE_MD_CHARS_MATCHER = re.compile(r"([\\\[\]\(\)])")
-RE_MD_CHARS_MATCHER_ALL = re.compile(r"([`\*_{}\[\]\(\)#!])")
-
-# to find links in the text
-RE_LINK = re.compile(r"(\[.*?\] ?\(.*?\))|(\[.*?\]:.*?)")
-
-# to find table separators
-RE_TABLE = re.compile(r" \| ")
-
-RE_MD_DOT_MATCHER = re.compile(
-    r"""
-    ^             # start of line
-    (\s*\d+)      # optional whitespace and a number
-    (\.)          # dot
-    (?=\s)        # lookahead assert whitespace
-    """,
-    re.MULTILINE | re.VERBOSE,
-)
-RE_MD_PLUS_MATCHER = re.compile(
-    r"""
-    ^
-    (\s*)
-    (\+)
-    (?=\s)
-    """,
-    flags=re.MULTILINE | re.VERBOSE,
-)
-RE_MD_DASH_MATCHER = re.compile(
-    r"""
-    ^
-    (\s*)
-    (-)
-    (?=\s|\-)     # followed by whitespace (bullet list, or spaced out hr)
-                  # or another dash (header or hr)
-    """,
-    flags=re.MULTILINE | re.VERBOSE,
-)
-RE_SLASH_CHARS = r"\`*_{}[]()#+-.!"
-RE_MD_BACKSLASH_MATCHER = re.compile(
-    r"""
-    (\\)          # match one slash
-    (?=[%s])      # followed by a char that requires escaping
-    """
-    % re.escape(RE_SLASH_CHARS),
-    flags=re.VERBOSE,
-)
-
-UNIFIABLE = {
-    "rsquo": "'",
-    "lsquo": "'",
-    "rdquo": '"',
-    "ldquo": '"',
-    "copy": "(C)",
-    "mdash": "--",
-    "nbsp": " ",
-    "rarr": "->",
-    "larr": "<-",
-    "middot": "*",
-    "ndash": "-",
-    "oelig": "oe",
-    "aelig": "ae",
-    "agrave": "a",
-    "aacute": "a",
-    "acirc": "a",
-    "atilde": "a",
-    "auml": "a",
-    "aring": "a",
-    "egrave": "e",
-    "eacute": "e",
-    "ecirc": "e",
-    "euml": "e",
-    "igrave": "i",
-    "iacute": "i",
-    "icirc": "i",
-    "iuml": "i",
-    "ograve": "o",
-    "oacute": "o",
-    "ocirc": "o",
-    "otilde": "o",
-    "ouml": "o",
-    "ugrave": "u",
-    "uacute": "u",
-    "ucirc": "u",
-    "uuml": "u",
-    "lrm": "",
-    "rlm": "",
-}
-
-# Format tables in HTML rather than Markdown syntax
-BYPASS_TABLES = False
-# Ignore table-related tags (table, th, td, tr) while keeping rows
-IGNORE_TABLES = False
-
-
-# Use a single line break after a block element rather than two line breaks.
-# NOTE: Requires body width setting to be 0.
-SINGLE_LINE_BREAK = False
-
-
-# Use double quotation marks when converting the <q> tag.
-OPEN_QUOTE = '"'
-CLOSE_QUOTE = '"'
-
-# Include the <sup> and <sub> tags
-INCLUDE_SUP_SUB = False

crawl4ai/html2text/elements.py

@@ -1,18 +0,0 @@
-from typing import Dict, Optional
-
-
-class AnchorElement:
-    __slots__ = ["attrs", "count", "outcount"]
-
-    def __init__(self, attrs: Dict[str, Optional[str]], count: int, outcount: int):
-        self.attrs = attrs
-        self.count = count
-        self.outcount = outcount
-
-
-class ListElement:
-    __slots__ = ["name", "num"]
-
-    def __init__(self, name: str, num: int):
-        self.name = name
-        self.num = num

crawl4ai/html2text/utils.py

@@ -1,304 +0,0 @@
-import html.entities
-from typing import Dict, List, Optional
-
-from . import config
-
-unifiable_n = {
-    html.entities.name2codepoint[k]: v
-    for k, v in config.UNIFIABLE.items()
-    if k != "nbsp"
-}
-
-
-def hn(tag: str) -> int:
-    if tag[0] == "h" and len(tag) == 2:
-        n = tag[1]
-        if "0" < n <= "9":
-            return int(n)
-    return 0
-
-
-def dumb_property_dict(style: str) -> Dict[str, str]:
-    """
-    :returns: A hash of css attributes
-    """
-    return {
-        x.strip().lower(): y.strip().lower()
-        for x, y in [z.split(":", 1) for z in style.split(";") if ":" in z]
-    }
-
-
-def dumb_css_parser(data: str) -> Dict[str, Dict[str, str]]:
-    """
-    :type data: str
-
-    :returns: A hash of css selectors, each of which contains a hash of
-    css attributes.
-    :rtype: dict
-    """
-    # remove @import sentences
-    data += ";"
-    importIndex = data.find("@import")
-    while importIndex != -1:
-        data = data[0:importIndex] + data[data.find(";", importIndex) + 1 :]
-        importIndex = data.find("@import")
-
-    # parse the css. reverted from dictionary comprehension in order to
-    # support older pythons
-    pairs = [x.split("{") for x in data.split("}") if "{" in x.strip()]
-    try:
-        elements = {a.strip(): dumb_property_dict(b) for a, b in pairs}
-    except ValueError:
-        elements = {}  # not that important
-
-    return elements
-
-
-def element_style(
-    attrs: Dict[str, Optional[str]],
-    style_def: Dict[str, Dict[str, str]],
-    parent_style: Dict[str, str],
-) -> Dict[str, str]:
-    """
-    :type attrs: dict
-    :type style_def: dict
-    :type style_def: dict
-
-    :returns: A hash of the 'final' style attributes of the element
-    :rtype: dict
-    """
-    style = parent_style.copy()
-    if "class" in attrs:
-        assert attrs["class"] is not None
-        for css_class in attrs["class"].split():
-            css_style = style_def.get("." + css_class, {})
-            style.update(css_style)
-    if "style" in attrs:
-        assert attrs["style"] is not None
-        immediate_style = dumb_property_dict(attrs["style"])
-        style.update(immediate_style)
-
-    return style
-
-
-def google_list_style(style: Dict[str, str]) -> str:
-    """
-    Finds out whether this is an ordered or unordered list
-
-    :type style: dict
-
-    :rtype: str
-    """
-    if "list-style-type" in style:
-        list_style = style["list-style-type"]
-        if list_style in ["disc", "circle", "square", "none"]:
-            return "ul"
-
-    return "ol"
-
-
-def google_has_height(style: Dict[str, str]) -> bool:
-    """
-    Check if the style of the element has the 'height' attribute
-    explicitly defined
-
-    :type style: dict
-
-    :rtype: bool
-    """
-    return "height" in style
-
-
-def google_text_emphasis(style: Dict[str, str]) -> List[str]:
-    """
-    :type style: dict
-
-    :returns: A list of all emphasis modifiers of the element
-    :rtype: list
-    """
-    emphasis = []
-    if "text-decoration" in style:
-        emphasis.append(style["text-decoration"])
-    if "font-style" in style:
-        emphasis.append(style["font-style"])
-    if "font-weight" in style:
-        emphasis.append(style["font-weight"])
-
-    return emphasis
-
-
-def google_fixed_width_font(style: Dict[str, str]) -> bool:
-    """
-    Check if the css of the current element defines a fixed width font
-
-    :type style: dict
-
-    :rtype: bool
-    """
-    font_family = ""
-    if "font-family" in style:
-        font_family = style["font-family"]
-    return "courier new" == font_family or "consolas" == font_family
-
-
-def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
-    """
-    Extract numbering from list element attributes
-
-    :type attrs: dict
-
-    :rtype: int or None
-    """
-    if "start" in attrs:
-        assert attrs["start"] is not None
-        try:
-            return int(attrs["start"]) - 1
-        except ValueError:
-            pass
-
-    return 0
-
-
-def skipwrap(
-    para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool
-) -> bool:
-    # If it appears to contain a link
-    # don't wrap
-    if not wrap_links and config.RE_LINK.search(para):
-        return True
-    # If the text begins with four spaces or one tab, it's a code block;
-    # don't wrap
-    if para[0:4] == "    " or para[0] == "\t":
-        return True
-
-    # If the text begins with only two "--", possibly preceded by
-    # whitespace, that's an emdash; so wrap.
-    stripped = para.lstrip()
-    if stripped[0:2] == "--" and len(stripped) > 2 and stripped[2] != "-":
-        return False
-
-    # I'm not sure what this is for; I thought it was to detect lists,
-    # but there's a <br>-inside-<span> case in one of the tests that
-    # also depends upon it.
-    if stripped[0:1] in ("-", "*") and not stripped[0:2] == "**":
-        return not wrap_list_items
-
-    # If text contains a pipe character it is likely a table
-    if not wrap_tables and config.RE_TABLE.search(para):
-        return True
-
-    # If the text begins with a single -, *, or +, followed by a space,
-    # or an integer, followed by a ., followed by a space (in either
-    # case optionally proceeded by whitespace), it's a list; don't wrap.
-    return bool(
-        config.RE_ORDERED_LIST_MATCHER.match(stripped)
-        or config.RE_UNORDERED_LIST_MATCHER.match(stripped)
-    )
-
-
-def escape_md(text: str) -> str:
-    """
-    Escapes markdown-sensitive characters within other markdown
-    constructs.
-    """
-    return config.RE_MD_CHARS_MATCHER.sub(r"\\\1", text)
-
-
-def escape_md_section(
-    text: str,
-    escape_backslash: bool = True,
-    snob: bool = False,
-    escape_dot: bool = True,
-    escape_plus: bool = True,
-    escape_dash: bool = True,
-) -> str:
-    """
-    Escapes markdown-sensitive characters across whole document sections.
-    Each escaping operation can be controlled individually.
-    """
-    if escape_backslash:
-        text = config.RE_MD_BACKSLASH_MATCHER.sub(r"\\\1", text)
-
-    if snob:
-        text = config.RE_MD_CHARS_MATCHER_ALL.sub(r"\\\1", text)
-
-    if escape_dot:
-        text = config.RE_MD_DOT_MATCHER.sub(r"\1\\\2", text)
-
-    if escape_plus:
-        text = config.RE_MD_PLUS_MATCHER.sub(r"\1\\\2", text)
-
-    if escape_dash:
-        text = config.RE_MD_DASH_MATCHER.sub(r"\1\\\2", text)
-
-    return text
-
-
-def reformat_table(lines: List[str], right_margin: int) -> List[str]:
-    """
-    Given the lines of a table
-    padds the cells and returns the new lines
-    """
-    # find the maximum width of the columns
-    max_width = [len(x.rstrip()) + right_margin for x in lines[0].split("|")]
-    max_cols = len(max_width)
-    for line in lines:
-        cols = [x.rstrip() for x in line.split("|")]
-        num_cols = len(cols)
-
-        # don't drop any data if colspan attributes result in unequal lengths
-        if num_cols < max_cols:
-            cols += [""] * (max_cols - num_cols)
-        elif max_cols < num_cols:
-            max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
-            max_cols = num_cols
-
-        max_width = [
-            max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
-        ]
-
-    # reformat
-    new_lines = []
-    for line in lines:
-        cols = [x.rstrip() for x in line.split("|")]
-        if set(line.strip()) == set("-|"):
-            filler = "-"
-            new_cols = [
-                x.rstrip() + (filler * (M - len(x.rstrip())))
-                for x, M in zip(cols, max_width)
-            ]
-            new_lines.append("|-" + "|".join(new_cols) + "|")
-        else:
-            filler = " "
-            new_cols = [
-                x.rstrip() + (filler * (M - len(x.rstrip())))
-                for x, M in zip(cols, max_width)
-            ]
-            new_lines.append("| " + "|".join(new_cols) + "|")
-    return new_lines
-
-
-def pad_tables_in_text(text: str, right_margin: int = 1) -> str:
-    """
-    Provide padding for tables in the text
-    """
-    lines = text.split("\n")
-    table_buffer = []  # type: List[str]
-    table_started = False
-    new_lines = []
-    for line in lines:
-        # Toggle table started
-        if config.TABLE_MARKER_FOR_PAD in line:
-            table_started = not table_started
-            if not table_started:
-                table = reformat_table(table_buffer, right_margin)
-                new_lines.extend(table)
-                table_buffer = []
-                new_lines.append("")
-            continue
-        # Process lines
-        if table_started:
-            table_buffer.append(line)
-        else:
-            new_lines.append(line)
-    return "\n".join(new_lines)

crawl4ai/hub.py

@@ -1,69 +0,0 @@
-# crawl4ai/hub.py
-from abc import ABC, abstractmethod
-from typing import Dict, Type, Union
-import logging
-import importlib
-from pathlib import Path
-import inspect
-
-logger = logging.getLogger(__name__)
-
-
-class BaseCrawler(ABC):
-    def __init__(self):
-        self.logger = logging.getLogger(self.__class__.__name__)
-        
-    @abstractmethod
-    async def run(self, url: str = "", **kwargs) -> str:
-        """
-        Implement this method to return JSON string.
-        Must accept URL + arbitrary kwargs for flexibility.
-        """
-        pass
-
-    def __init_subclass__(cls, **kwargs):
-        """Enforce interface validation on subclassing"""
-        super().__init_subclass__(**kwargs)
-        
-        # Verify run method signature
-        run_method = cls.run
-        if not run_method.__code__.co_argcount >= 2:  # self + url
-            raise TypeError(f"{cls.__name__} must implement 'run(self, url: str, **kwargs)'")
-            
-        # Verify async nature
-        if not inspect.iscoroutinefunction(run_method):
-            raise TypeError(f"{cls.__name__}.run must be async")
-
-class CrawlerHub:
-    _crawlers: Dict[str, Type[BaseCrawler]] = {}
-
-    @classmethod
-    def _discover_crawlers(cls):
-        """Dynamically load crawlers from /crawlers in 3 lines"""
-        base_path = Path(__file__).parent / "crawlers"
-        for crawler_dir in base_path.iterdir():
-            if crawler_dir.is_dir():
-                try:
-                    module = importlib.import_module(
-                        f"crawl4ai.crawlers.{crawler_dir.name}.crawler"
-                    )
-                    for attr in dir(module):
-                        cls._maybe_register_crawler(
-                            getattr(module, attr), crawler_dir.name
-                        )
-                except Exception as e:
-                    logger.warning(f"Failed {crawler_dir.name}: {str(e)}")
-
-    @classmethod
-    def _maybe_register_crawler(cls, obj, name: str):
-        """Brilliant one-liner registration"""
-        if isinstance(obj, type) and issubclass(obj, BaseCrawler) and obj != BaseCrawler:
-            module = importlib.import_module(obj.__module__)
-            obj.meta = getattr(module, "__meta__", {})
-            cls._crawlers[name] = obj
-
-    @classmethod
-    def get(cls, name: str) -> Union[Type[BaseCrawler], None]:
-        if not cls._crawlers:
-            cls._discover_crawlers()
-        return cls._crawlers.get(name)

crawl4ai/install.py

@@ -1,212 +0,0 @@
-import subprocess
-import sys
-import asyncio
-from .async_logger import AsyncLogger, LogLevel
-from pathlib import Path
-import os
-import shutil
-
-# Initialize logger
-logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
-
-def setup_home_directory():
-    """Set up the .crawl4ai folder structure in the user's home directory."""
-    base_dir = os.getenv("CRAWL4_AI_BASE_DIRECTORY")
-    crawl4ai_folder = Path(base_dir) if base_dir else Path.home()
-    crawl4ai_config = crawl4ai_folder / "global.yml"
-    crawl4ai_folder = crawl4ai_folder / ".crawl4ai"
-    cache_folder = crawl4ai_folder / "cache"
-    content_folders = [
-        "html_content",
-        "cleaned_html",
-        "markdown_content",
-        "extracted_content",
-        "screenshots",
-    ]
-
-    # Clean up old cache if exists
-    if cache_folder.exists():
-        shutil.rmtree(cache_folder)
-
-    # Create new folder structure
-    crawl4ai_folder.mkdir(exist_ok=True)
-    cache_folder.mkdir(exist_ok=True)
-    for folder in content_folders:
-        (crawl4ai_folder / folder).mkdir(exist_ok=True)
-    
-    # If config file does not exist, create it
-    if not crawl4ai_config.exists():
-        with open(crawl4ai_config, "w") as f:
-            f.write("")
-
-def post_install():
-    """
-    Run all post-installation tasks.
-    Checks CRAWL4AI_MODE environment variable. If set to 'api',
-    skips Playwright browser installation.
-    """
-    logger.info("Running post-installation setup...", tag="INIT")
-    setup_home_directory()
-
-    # Check environment variable to conditionally skip Playwright install
-    run_mode = os.getenv('CRAWL4AI_MODE')
-    if run_mode == 'api':
-        logger.warning(
-            "CRAWL4AI_MODE=api detected. Skipping Playwright browser installation.",
-            tag="SETUP"
-        )
-    else:
-        # Proceed with installation only if mode is not 'api'
-        install_playwright()
-
-    run_migration()
-    # TODO: Will be added in the future
-    # setup_builtin_browser()
-    logger.success("Post-installation setup completed!", tag="COMPLETE")
-    
-def setup_builtin_browser():
-    """Set up a builtin browser for use with Crawl4AI"""
-    try:
-        logger.info("Setting up builtin browser...", tag="INIT")
-        asyncio.run(_setup_builtin_browser())
-        logger.success("Builtin browser setup completed!", tag="COMPLETE")
-    except Exception as e:
-        logger.warning(f"Failed to set up builtin browser: {e}")
-        logger.warning("You can manually set up a builtin browser using 'crawl4ai-doctor builtin-browser-start'")
-    
-async def _setup_builtin_browser():
-    try:
-        # Import BrowserProfiler here to avoid circular imports
-        from .browser_profiler import BrowserProfiler
-        profiler = BrowserProfiler(logger=logger)
-        
-        # Launch the builtin browser
-        cdp_url = await profiler.launch_builtin_browser(headless=True)
-        if cdp_url:
-            logger.success(f"Builtin browser launched at {cdp_url}", tag="BROWSER")
-        else:
-            logger.warning("Failed to launch builtin browser", tag="BROWSER")
-    except Exception as e:
-        logger.warning(f"Error setting up builtin browser: {e}", tag="BROWSER")
-        raise
-
-
-def install_playwright():
-    logger.info("Installing Playwright browsers...", tag="INIT")
-    try:
-        # subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chrome"])
-        subprocess.check_call(
-            [
-                sys.executable,
-                "-m",
-                "playwright",
-                "install",
-                "--with-deps",
-                "--force",
-                "chromium",
-            ]
-        )
-        logger.success(
-            "Playwright installation completed successfully.", tag="COMPLETE"
-        )
-    except subprocess.CalledProcessError:
-        # logger.error(f"Error during Playwright installation: {e}", tag="ERROR")
-        logger.warning(
-            f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
-        )
-    except Exception:
-        # logger.error(f"Unexpected error during Playwright installation: {e}", tag="ERROR")
-        logger.warning(
-            f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation."
-        )
-    
-    # Install Patchright browsers for undetected browser support
-    logger.info("Installing Patchright browsers for undetected mode...", tag="INIT")
-    try:
-        subprocess.check_call(
-            [
-                sys.executable,
-                "-m",
-                "patchright",
-                "install",
-                "--with-deps",
-                "--force",
-                "chromium",
-            ]
-        )
-        logger.success(
-            "Patchright installation completed successfully.", tag="COMPLETE"
-        )
-    except subprocess.CalledProcessError:
-        logger.warning(
-            f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."
-        )
-    except Exception:
-        logger.warning(
-            f"Please run '{sys.executable} -m patchright install --with-deps' manually after the installation."
-        )
-
-
-def run_migration():
-    """Initialize database during installation"""
-    try:
-        logger.info("Starting database initialization...", tag="INIT")
-        from crawl4ai.async_database import async_db_manager
-
-        asyncio.run(async_db_manager.initialize())
-        logger.success(
-            "Database initialization completed successfully.", tag="COMPLETE"
-        )
-    except ImportError:
-        logger.warning("Database module not found. Will initialize on first use.")
-    except Exception as e:
-        logger.warning(f"Database initialization failed: {e}")
-        logger.warning("Database will be initialized on first use")
-
-
-async def run_doctor():
-    """Test if Crawl4AI is working properly"""
-    logger.info("Running Crawl4AI health check...", tag="INIT")
-    try:
-        from .async_webcrawler import (
-            AsyncWebCrawler,
-            BrowserConfig,
-            CrawlerRunConfig,
-            CacheMode,
-        )
-
-        browser_config = BrowserConfig(
-            headless=True,
-            browser_type="chromium",
-            ignore_https_errors=True,
-            light_mode=True,
-            viewport_width=1280,
-            viewport_height=720,
-        )
-
-        run_config = CrawlerRunConfig(
-            cache_mode=CacheMode.BYPASS,
-            screenshot=True,
-        )
-
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            logger.info("Testing crawling capabilities...", tag="TEST")
-            result = await crawler.arun(url="https://crawl4ai.com", config=run_config)
-
-            if result and result.markdown:
-                logger.success("✅ Crawling test passed!", tag="COMPLETE")
-                return True
-            else:
-                raise Exception("Failed to get content")
-
-    except Exception as e:
-        logger.error(f"❌ Test failed: {e}", tag="ERROR")
-        return False
-
-
-def doctor():
-    """Entry point for the doctor command"""
-    import asyncio
-
-    asyncio.run(run_doctor())
-    sys.exit(0)

crawl4ai/js_snippet/__init__.py

@@ -1,18 +0,0 @@
-import os
-
-
-# Create a function get name of a js script, then load from the CURRENT folder of this script and return its content as string, make sure its error free
-def load_js_script(script_name):
-    # Get the path of the current script
-    current_script_path = os.path.dirname(os.path.realpath(__file__))
-    # Get the path of the script to load
-    script_path = os.path.join(current_script_path, script_name + ".js")
-    # Check if the script exists
-    if not os.path.exists(script_path):
-        raise ValueError(
-            f"Script {script_name} not found in the folder {current_script_path}"
-        )
-    # Load the content of the script
-    with open(script_path, "r") as f:
-        script_content = f.read()
-    return script_content

crawl4ai/js_snippet/navigator_overrider.js

@@ -1,25 +0,0 @@
-// Pass the Permissions Test.
-const originalQuery = window.navigator.permissions.query;
-window.navigator.permissions.query = (parameters) =>
-    parameters.name === "notifications"
-        ? Promise.resolve({ state: Notification.permission })
-        : originalQuery(parameters);
-Object.defineProperty(navigator, "webdriver", {
-    get: () => undefined,
-});
-window.navigator.chrome = {
-    runtime: {},
-    // Add other properties if necessary
-};
-Object.defineProperty(navigator, "plugins", {
-    get: () => [1, 2, 3, 4, 5],
-});
-Object.defineProperty(navigator, "languages", {
-    get: () => ["en-US", "en"],
-});
-Object.defineProperty(document, "hidden", {
-    get: () => false,
-});
-Object.defineProperty(document, "visibilityState", {
-    get: () => "visible",
-});

crawl4ai/js_snippet/remove_overlay_elements.js

@@ -1,120 +0,0 @@
-async () => {
-    // Function to check if element is visible
-    const isVisible = (elem) => {
-        const style = window.getComputedStyle(elem);
-        return style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0";
-    };
-
-    // Common selectors for popups and overlays
-    const commonSelectors = [
-        // Close buttons first
-        'button[class*="close" i]',
-        'button[class*="dismiss" i]',
-        'button[aria-label*="close" i]',
-        'button[title*="close" i]',
-        'a[class*="close" i]',
-        'span[class*="close" i]',
-
-        // Cookie notices
-        '[class*="cookie-banner" i]',
-        '[id*="cookie-banner" i]',
-        '[class*="cookie-consent" i]',
-        '[id*="cookie-consent" i]',
-
-        // Newsletter/subscription dialogs
-        '[class*="newsletter" i]',
-        '[class*="subscribe" i]',
-
-        // Generic popups/modals
-        '[class*="popup" i]',
-        '[class*="modal" i]',
-        '[class*="overlay" i]',
-        '[class*="dialog" i]',
-        '[role="dialog"]',
-        '[role="alertdialog"]',
-    ];
-
-    // Try to click close buttons first
-    for (const selector of commonSelectors.slice(0, 6)) {
-        const closeButtons = document.querySelectorAll(selector);
-        for (const button of closeButtons) {
-            if (isVisible(button)) {
-                try {
-                    button.click();
-                    await new Promise((resolve) => setTimeout(resolve, 100));
-                } catch (e) {
-                    console.log("Error clicking button:", e);
-                }
-            }
-        }
-    }
-
-    // Remove remaining overlay elements
-    const removeOverlays = () => {
-        // Find elements with high z-index
-        const allElements = document.querySelectorAll("*");
-        for (const elem of allElements) {
-            const style = window.getComputedStyle(elem);
-            const zIndex = parseInt(style.zIndex);
-            const position = style.position;
-
-            if (
-                isVisible(elem) &&
-                (zIndex > 999 || position === "fixed" || position === "absolute") &&
-                (elem.offsetWidth > window.innerWidth * 0.5 ||
-                    elem.offsetHeight > window.innerHeight * 0.5 ||
-                    style.backgroundColor.includes("rgba") ||
-                    parseFloat(style.opacity) < 1)
-            ) {
-                elem.remove();
-            }
-        }
-
-        // Remove elements matching common selectors
-        for (const selector of commonSelectors) {
-            const elements = document.querySelectorAll(selector);
-            elements.forEach((elem) => {
-                if (isVisible(elem)) {
-                    elem.remove();
-                }
-            });
-        }
-    };
-
-    // Remove overlay elements
-    removeOverlays();
-
-    // Remove any fixed/sticky position elements at the top/bottom
-    const removeFixedElements = () => {
-        const elements = document.querySelectorAll("*");
-        elements.forEach((elem) => {
-            const style = window.getComputedStyle(elem);
-            if ((style.position === "fixed" || style.position === "sticky") && isVisible(elem)) {
-                elem.remove();
-            }
-        });
-    };
-
-    removeFixedElements();
-
-    // Remove empty block elements as: div, p, span, etc.
-    const removeEmptyBlockElements = () => {
-        const blockElements = document.querySelectorAll(
-            "div, p, span, section, article, header, footer, aside, nav, main, ul, ol, li, dl, dt, dd, h1, h2, h3, h4, h5, h6"
-        );
-        blockElements.forEach((elem) => {
-            if (elem.innerText.trim() === "") {
-                elem.remove();
-            }
-        });
-    };
-
-    // Remove margin-right and padding-right from body (often added by modal scripts)
-    document.body.style.marginRight = "0px";
-    document.body.style.paddingRight = "0px";
-    document.body.style.overflow = "auto";
-
-    // Wait a bit for any animations to complete
-    document.body.scrollIntoView(false);
-    await new Promise((resolve) => setTimeout(resolve, 50));
-};

crawl4ai/js_snippet/update_image_dimensions.js

@@ -1,54 +0,0 @@
-() => {
-    return new Promise((resolve) => {
-        const filterImage = (img) => {
-            // Filter out images that are too small
-            if (img.width < 100 && img.height < 100) return false;
-
-            // Filter out images that are not visible
-            const rect = img.getBoundingClientRect();
-            if (rect.width === 0 || rect.height === 0) return false;
-
-            // Filter out images with certain class names (e.g., icons, thumbnails)
-            if (img.classList.contains("icon") || img.classList.contains("thumbnail")) return false;
-
-            // Filter out images with certain patterns in their src (e.g., placeholder images)
-            if (img.src.includes("placeholder") || img.src.includes("icon")) return false;
-
-            return true;
-        };
-
-        const images = Array.from(document.querySelectorAll("img")).filter(filterImage);
-        let imagesLeft = images.length;
-
-        if (imagesLeft === 0) {
-            resolve();
-            return;
-        }
-
-        const checkImage = (img) => {
-            if (img.complete && img.naturalWidth !== 0) {
-                img.setAttribute("width", img.naturalWidth);
-                img.setAttribute("height", img.naturalHeight);
-                imagesLeft--;
-                if (imagesLeft === 0) resolve();
-            }
-        };
-
-        images.forEach((img) => {
-            checkImage(img);
-            if (!img.complete) {
-                img.onload = () => {
-                    checkImage(img);
-                };
-                img.onerror = () => {
-                    imagesLeft--;
-                    if (imagesLeft === 0) resolve();
-                };
-            }
-        });
-
-        // Fallback timeout of 5 seconds
-        // setTimeout(() => resolve(), 5000);
-        resolve();
-    });
-};

crawl4ai/legacy/cli.py

@@ -1,123 +0,0 @@
-import click
-import sys
-import asyncio
-from typing import List
-from .docs_manager import DocsManager
-from .async_logger import AsyncLogger
-
-logger = AsyncLogger(verbose=True)
-docs_manager = DocsManager(logger)
-
-
-def print_table(headers: List[str], rows: List[List[str]], padding: int = 2):
-    """Print formatted table with headers and rows"""
-    widths = [max(len(str(cell)) for cell in col) for col in zip(headers, *rows)]
-    border = "+" + "+".join("-" * (w + 2 * padding) for w in widths) + "+"
-
-    def format_row(row):
-        return (
-            "|"
-            + "|".join(
-                f"{' ' * padding}{str(cell):<{w}}{' ' * padding}"
-                for cell, w in zip(row, widths)
-            )
-            + "|"
-        )
-
-    click.echo(border)
-    click.echo(format_row(headers))
-    click.echo(border)
-    for row in rows:
-        click.echo(format_row(row))
-    click.echo(border)
-
-
-@click.group()
-def cli():
-    """Crawl4AI Command Line Interface"""
-    pass
-
-
-@cli.group()
-def docs():
-    """Documentation operations"""
-    pass
-
-
-@docs.command()
-@click.argument("sections", nargs=-1)
-@click.option(
-    "--mode", type=click.Choice(["extended", "condensed"]), default="extended"
-)
-def combine(sections: tuple, mode: str):
-    """Combine documentation sections"""
-    try:
-        asyncio.run(docs_manager.ensure_docs_exist())
-        click.echo(docs_manager.generate(sections, mode))
-    except Exception as e:
-        logger.error(str(e), tag="ERROR")
-        sys.exit(1)
-
-
-@docs.command()
-@click.argument("query")
-@click.option("--top-k", "-k", default=5)
-@click.option("--build-index", is_flag=True, help="Build index if missing")
-def search(query: str, top_k: int, build_index: bool):
-    """Search documentation"""
-    try:
-        result = docs_manager.search(query, top_k)
-        if result == "No search index available. Call build_search_index() first.":
-            if build_index or click.confirm("No search index found. Build it now?"):
-                asyncio.run(docs_manager.llm_text.generate_index_files())
-                result = docs_manager.search(query, top_k)
-        click.echo(result)
-    except Exception as e:
-        click.echo(f"Error: {str(e)}", err=True)
-        sys.exit(1)
-
-
-@docs.command()
-def update():
-    """Update docs from GitHub"""
-    try:
-        asyncio.run(docs_manager.fetch_docs())
-        click.echo("Documentation updated successfully")
-    except Exception as e:
-        click.echo(f"Error: {str(e)}", err=True)
-        sys.exit(1)
-
-
-@docs.command()
-@click.option("--force-facts", is_flag=True, help="Force regenerate fact files")
-@click.option("--clear-cache", is_flag=True, help="Clear BM25 cache")
-def index(force_facts: bool, clear_cache: bool):
-    """Build or rebuild search indexes"""
-    try:
-        asyncio.run(docs_manager.ensure_docs_exist())
-        asyncio.run(
-            docs_manager.llm_text.generate_index_files(
-                force_generate_facts=force_facts, clear_bm25_cache=clear_cache
-            )
-        )
-        click.echo("Search indexes built successfully")
-    except Exception as e:
-        click.echo(f"Error: {str(e)}", err=True)
-        sys.exit(1)
-
-
-# Add docs list command
-@docs.command()
-def list():
-    """List available documentation sections"""
-    try:
-        sections = docs_manager.list()
-        print_table(["Sections"], [[section] for section in sections])
-
-    except Exception as e:
-        click.echo(f"Error: {str(e)}", err=True)
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    cli()

crawl4ai/legacy/docs_manager.py

@@ -1,75 +0,0 @@
-import requests
-import shutil
-from pathlib import Path
-from crawl4ai.async_logger import AsyncLogger
-from crawl4ai.llmtxt import AsyncLLMTextManager
-
-
-class DocsManager:
-    def __init__(self, logger=None):
-        self.docs_dir = Path.home() / ".crawl4ai" / "docs"
-        self.local_docs = Path(__file__).parent.parent / "docs" / "llm.txt"
-        self.docs_dir.mkdir(parents=True, exist_ok=True)
-        self.logger = logger or AsyncLogger(verbose=True)
-        self.llm_text = AsyncLLMTextManager(self.docs_dir, self.logger)
-
-    async def ensure_docs_exist(self):
-        """Fetch docs if not present"""
-        if not any(self.docs_dir.iterdir()):
-            await self.fetch_docs()
-
-    async def fetch_docs(self) -> bool:
-        """Copy from local docs or download from GitHub"""
-        try:
-            # Try local first
-            if self.local_docs.exists() and (
-                any(self.local_docs.glob("*.md"))
-                or any(self.local_docs.glob("*.tokens"))
-            ):
-                # Empty the local docs directory
-                for file_path in self.docs_dir.glob("*.md"):
-                    file_path.unlink()
-                # for file_path in self.docs_dir.glob("*.tokens"):
-                #     file_path.unlink()
-                for file_path in self.local_docs.glob("*.md"):
-                    shutil.copy2(file_path, self.docs_dir / file_path.name)
-                # for file_path in self.local_docs.glob("*.tokens"):
-                #     shutil.copy2(file_path, self.docs_dir / file_path.name)
-                return True
-
-            # Fallback to GitHub
-            response = requests.get(
-                "https://api.github.com/repos/unclecode/crawl4ai/contents/docs/llm.txt",
-                headers={"Accept": "application/vnd.github.v3+json"},
-            )
-            response.raise_for_status()
-
-            for item in response.json():
-                if item["type"] == "file" and item["name"].endswith(".md"):
-                    content = requests.get(item["download_url"]).text
-                    with open(self.docs_dir / item["name"], "w", encoding="utf-8") as f:
-                        f.write(content)
-            return True
-
-        except Exception as e:
-            self.logger.error(f"Failed to fetch docs: {str(e)}")
-            raise
-
-    def list(self) -> list[str]:
-        """List available topics"""
-        names = [file_path.stem for file_path in self.docs_dir.glob("*.md")]
-        # Remove [0-9]+_ prefix
-        names = [name.split("_", 1)[1] if name[0].isdigit() else name for name in names]
-        # Exclude those end with .xs.md and .q.md
-        names = [
-            name
-            for name in names
-            if not name.endswith(".xs") and not name.endswith(".q")
-        ]
-        return names
-
-    def generate(self, sections, mode="extended"):
-        return self.llm_text.generate(sections, mode)
-
-    def search(self, query: str, top_k: int = 5):
-        return self.llm_text.search(query, top_k)

crawl4ai/legacy/llmtxt.py

@@ -1,546 +0,0 @@
-import os
-from pathlib import Path
-import re
-from typing import Dict, List, Tuple, Optional, Any
-import json
-from tqdm import tqdm
-import time
-import psutil
-import numpy as np
-from rank_bm25 import BM25Okapi
-from nltk.tokenize import word_tokenize
-from nltk.corpus import stopwords
-from nltk.stem import WordNetLemmatizer
-from litellm import batch_completion
-from .async_logger import AsyncLogger
-import litellm
-import pickle
-import hashlib  # <--- ADDED for file-hash
-import glob
-
-litellm.set_verbose = False
-
-
-def _compute_file_hash(file_path: Path) -> str:
-    """Compute MD5 hash for the file's entire content."""
-    hash_md5 = hashlib.md5()
-    with file_path.open("rb") as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            hash_md5.update(chunk)
-    return hash_md5.hexdigest()
-
-
-class AsyncLLMTextManager:
-    def __init__(
-        self,
-        docs_dir: Path,
-        logger: Optional[AsyncLogger] = None,
-        max_concurrent_calls: int = 5,
-        batch_size: int = 3,
-    ) -> None:
-        self.docs_dir = docs_dir
-        self.logger = logger
-        self.max_concurrent_calls = max_concurrent_calls
-        self.batch_size = batch_size
-        self.bm25_index = None
-        self.document_map: Dict[str, Any] = {}
-        self.tokenized_facts: List[str] = []
-        self.bm25_index_file = self.docs_dir / "bm25_index.pkl"
-
-    async def _process_document_batch(self, doc_batch: List[Path]) -> None:
-        """Process a batch of documents in parallel"""
-        contents = []
-        for file_path in doc_batch:
-            try:
-                with open(file_path, "r", encoding="utf-8") as f:
-                    contents.append(f.read())
-            except Exception as e:
-                self.logger.error(f"Error reading {file_path}: {str(e)}")
-                contents.append("")  # Add empty content to maintain batch alignment
-
-        prompt = """Given a documentation file, generate a list of atomic facts where each fact:
-1. Represents a single piece of knowledge
-2. Contains variations in terminology for the same concept
-3. References relevant code patterns if they exist
-4. Is written in a way that would match natural language queries
-
-Each fact should follow this format:
-<main_concept>: <fact_statement> | <related_terms> | <code_reference>
-
-Example Facts:
-browser_config: Configure headless mode and browser type for AsyncWebCrawler | headless, browser_type, chromium, firefox | BrowserConfig(browser_type="chromium", headless=True)
-redis_connection: Redis client connection requires host and port configuration | redis setup, redis client, connection params | Redis(host='localhost', port=6379, db=0)
-pandas_filtering: Filter DataFrame rows using boolean conditions | dataframe filter, query, boolean indexing | df[df['column'] > 5]
-
-Wrap your response in <index>...</index> tags.
-"""
-
-        # Prepare messages for batch processing
-        messages_list = [
-            [
-                {
-                    "role": "user",
-                    "content": f"{prompt}\n\nGenerate index for this documentation:\n\n{content}",
-                }
-            ]
-            for content in contents
-            if content
-        ]
-
-        try:
-            responses = batch_completion(
-                model="anthropic/claude-3-5-sonnet-latest",
-                messages=messages_list,
-                logger_fn=None,
-            )
-
-            # Process responses and save index files
-            for response, file_path in zip(responses, doc_batch):
-                try:
-                    index_content_match = re.search(
-                        r"<index>(.*?)</index>",
-                        response.choices[0].message.content,
-                        re.DOTALL,
-                    )
-                    if not index_content_match:
-                        self.logger.warning(
-                            f"No <index>...</index> content found for {file_path}"
-                        )
-                        continue
-
-                    index_content = re.sub(
-                        r"\n\s*\n", "\n", index_content_match.group(1)
-                    ).strip()
-                    if index_content:
-                        index_file = file_path.with_suffix(".q.md")
-                        with open(index_file, "w", encoding="utf-8") as f:
-                            f.write(index_content)
-                        self.logger.info(f"Created index file: {index_file}")
-                    else:
-                        self.logger.warning(
-                            f"No index content found in response for {file_path}"
-                        )
-
-                except Exception as e:
-                    self.logger.error(
-                        f"Error processing response for {file_path}: {str(e)}"
-                    )
-
-        except Exception as e:
-            self.logger.error(f"Error in batch completion: {str(e)}")
-
-    def _validate_fact_line(self, line: str) -> Tuple[bool, Optional[str]]:
-        if "|" not in line:
-            return False, "Missing separator '|'"
-
-        parts = [p.strip() for p in line.split("|")]
-        if len(parts) != 3:
-            return False, f"Expected 3 parts, got {len(parts)}"
-
-        concept_part = parts[0]
-        if ":" not in concept_part:
-            return False, "Missing ':' in concept definition"
-
-        return True, None
-
-    def _load_or_create_token_cache(self, fact_file: Path) -> Dict:
-        """
-        Load token cache from .q.tokens if present and matching file hash.
-        Otherwise return a new structure with updated file-hash.
-        """
-        cache_file = fact_file.with_suffix(".q.tokens")
-        current_hash = _compute_file_hash(fact_file)
-
-        if cache_file.exists():
-            try:
-                with open(cache_file, "r") as f:
-                    cache = json.load(f)
-                # If the hash matches, return it directly
-                if cache.get("content_hash") == current_hash:
-                    return cache
-                # Otherwise, we signal that it's changed
-                self.logger.info(f"Hash changed for {fact_file}, reindex needed.")
-            except json.JSONDecodeError:
-                self.logger.warning(f"Corrupt token cache for {fact_file}, rebuilding.")
-            except Exception as e:
-                self.logger.warning(f"Error reading cache for {fact_file}: {str(e)}")
-
-        # Return a fresh cache
-        return {"facts": {}, "content_hash": current_hash}
-
-    def _save_token_cache(self, fact_file: Path, cache: Dict) -> None:
-        cache_file = fact_file.with_suffix(".q.tokens")
-        # Always ensure we're saving the correct file-hash
-        cache["content_hash"] = _compute_file_hash(fact_file)
-        with open(cache_file, "w") as f:
-            json.dump(cache, f)
-
-    def preprocess_text(self, text: str) -> List[str]:
-        parts = [x.strip() for x in text.split("|")] if "|" in text else [text]
-        # Remove : after the first word of parts[0]
-        parts[0] = re.sub(r"^(.*?):", r"\1", parts[0])
-
-        lemmatizer = WordNetLemmatizer()
-        stop_words = set(stopwords.words("english")) - {
-            "how",
-            "what",
-            "when",
-            "where",
-            "why",
-            "which",
-        }
-
-        tokens = []
-        for part in parts:
-            if "(" in part and ")" in part:
-                code_tokens = re.findall(
-                    r'[\w_]+(?=\()|[\w_]+(?==[\'"]{1}[\w_]+[\'"]{1})', part
-                )
-                tokens.extend(code_tokens)
-
-            words = word_tokenize(part.lower())
-            tokens.extend(
-                [
-                    lemmatizer.lemmatize(token)
-                    for token in words
-                    if token not in stop_words
-                ]
-            )
-
-        return tokens
-
-    def maybe_load_bm25_index(self, clear_cache=False) -> bool:
-        """
-        Load existing BM25 index from disk, if present and clear_cache=False.
-        """
-        if not clear_cache and os.path.exists(self.bm25_index_file):
-            self.logger.info("Loading existing BM25 index from disk.")
-            with open(self.bm25_index_file, "rb") as f:
-                data = pickle.load(f)
-            self.tokenized_facts = data["tokenized_facts"]
-            self.bm25_index = data["bm25_index"]
-            return True
-        return False
-
-    def build_search_index(self, clear_cache=False) -> None:
-        """
-        Checks for new or modified .q.md files by comparing file-hash.
-        If none need reindexing and clear_cache is False, loads existing index if available.
-        Otherwise, reindexes only changed/new files and merges or creates a new index.
-        """
-        # If clear_cache is True, we skip partial logic: rebuild everything from scratch
-        if clear_cache:
-            self.logger.info("Clearing cache and rebuilding full search index.")
-            if self.bm25_index_file.exists():
-                self.bm25_index_file.unlink()
-
-        process = psutil.Process()
-        self.logger.info("Checking which .q.md files need (re)indexing...")
-
-        # Gather all .q.md files
-        q_files = [
-            self.docs_dir / f for f in os.listdir(self.docs_dir) if f.endswith(".q.md")
-        ]
-
-        # We'll store known (unchanged) facts in these lists
-        existing_facts: List[str] = []
-        existing_tokens: List[List[str]] = []
-
-        # Keep track of invalid lines for logging
-        invalid_lines = []
-        needSet = []  # files that must be (re)indexed
-
-        for qf in q_files:
-            token_cache_file = qf.with_suffix(".q.tokens")
-
-            # If no .q.tokens or clear_cache is True → definitely reindex
-            if clear_cache or not token_cache_file.exists():
-                needSet.append(qf)
-                continue
-
-            # Otherwise, load the existing cache and compare hash
-            cache = self._load_or_create_token_cache(qf)
-            # If the .q.tokens was out of date (i.e. changed hash), we reindex
-            if len(cache["facts"]) == 0 or cache.get(
-                "content_hash"
-            ) != _compute_file_hash(qf):
-                needSet.append(qf)
-            else:
-                # File is unchanged → retrieve cached token data
-                for line, cache_data in cache["facts"].items():
-                    existing_facts.append(line)
-                    existing_tokens.append(cache_data["tokens"])
-                    self.document_map[line] = qf  # track the doc for that fact
-
-        if not needSet and not clear_cache:
-            # If no file needs reindexing, try loading existing index
-            if self.maybe_load_bm25_index(clear_cache=False):
-                self.logger.info(
-                    "No new/changed .q.md files found. Using existing BM25 index."
-                )
-                return
-            else:
-                # If there's no existing index, we must build a fresh index from the old caches
-                self.logger.info(
-                    "No existing BM25 index found. Building from cached facts."
-                )
-                if existing_facts:
-                    self.logger.info(
-                        f"Building BM25 index with {len(existing_facts)} cached facts."
-                    )
-                    self.bm25_index = BM25Okapi(existing_tokens)
-                    self.tokenized_facts = existing_facts
-                    with open(self.bm25_index_file, "wb") as f:
-                        pickle.dump(
-                            {
-                                "bm25_index": self.bm25_index,
-                                "tokenized_facts": self.tokenized_facts,
-                            },
-                            f,
-                        )
-                else:
-                    self.logger.warning("No facts found at all. Index remains empty.")
-                return
-
-        # ----------------------------------------------------- /Users/unclecode/.crawl4ai/docs/14_proxy_security.q.q.tokens '/Users/unclecode/.crawl4ai/docs/14_proxy_security.q.md'
-        # If we reach here, we have new or changed .q.md files
-        # We'll parse them, reindex them, and then combine with existing_facts
-        # -----------------------------------------------------
-
-        self.logger.info(f"{len(needSet)} file(s) need reindexing. Parsing now...")
-
-        # 1) Parse the new or changed .q.md files
-        new_facts = []
-        new_tokens = []
-        with tqdm(total=len(needSet), desc="Indexing changed files") as file_pbar:
-            for file in needSet:
-                # We'll build up a fresh cache
-                fresh_cache = {"facts": {}, "content_hash": _compute_file_hash(file)}
-                try:
-                    with open(file, "r", encoding="utf-8") as f_obj:
-                        content = f_obj.read().strip()
-                        lines = [l.strip() for l in content.split("\n") if l.strip()]
-
-                    for line in lines:
-                        is_valid, error = self._validate_fact_line(line)
-                        if not is_valid:
-                            invalid_lines.append((file, line, error))
-                            continue
-
-                        tokens = self.preprocess_text(line)
-                        fresh_cache["facts"][line] = {
-                            "tokens": tokens,
-                            "added": time.time(),
-                        }
-                        new_facts.append(line)
-                        new_tokens.append(tokens)
-                        self.document_map[line] = file
-
-                    # Save the new .q.tokens with updated hash
-                    self._save_token_cache(file, fresh_cache)
-
-                    mem_usage = process.memory_info().rss / 1024 / 1024
-                    self.logger.debug(
-                        f"Memory usage after {file.name}: {mem_usage:.2f}MB"
-                    )
-
-                except Exception as e:
-                    self.logger.error(f"Error processing {file}: {str(e)}")
-
-                file_pbar.update(1)
-
-        if invalid_lines:
-            self.logger.warning(f"Found {len(invalid_lines)} invalid fact lines:")
-            for file, line, error in invalid_lines:
-                self.logger.warning(f"{file}: {error} in line: {line[:50]}...")
-
-        # 2) Merge newly tokenized facts with the existing ones
-        all_facts = existing_facts + new_facts
-        all_tokens = existing_tokens + new_tokens
-
-        # 3) Build BM25 index from combined facts
-        self.logger.info(
-            f"Building BM25 index with {len(all_facts)} total facts (old + new)."
-        )
-        self.bm25_index = BM25Okapi(all_tokens)
-        self.tokenized_facts = all_facts
-
-        # 4) Save the updated BM25 index to disk
-        with open(self.bm25_index_file, "wb") as f:
-            pickle.dump(
-                {
-                    "bm25_index": self.bm25_index,
-                    "tokenized_facts": self.tokenized_facts,
-                },
-                f,
-            )
-
-        final_mem = process.memory_info().rss / 1024 / 1024
-        self.logger.info(f"Search index updated. Final memory usage: {final_mem:.2f}MB")
-
-    async def generate_index_files(
-        self, force_generate_facts: bool = False, clear_bm25_cache: bool = False
-    ) -> None:
-        """
-        Generate index files for all documents in parallel batches
-
-        Args:
-            force_generate_facts (bool): If True, regenerate indexes even if they exist
-            clear_bm25_cache (bool): If True, clear existing BM25 index cache
-        """
-        self.logger.info("Starting index generation for documentation files.")
-
-        md_files = [
-            self.docs_dir / f
-            for f in os.listdir(self.docs_dir)
-            if f.endswith(".md") and not any(f.endswith(x) for x in [".q.md", ".xs.md"])
-        ]
-
-        # Filter out files that already have .q files unless force=True
-        if not force_generate_facts:
-            md_files = [
-                f
-                for f in md_files
-                if not (self.docs_dir / f.name.replace(".md", ".q.md")).exists()
-            ]
-
-        if not md_files:
-            self.logger.info("All index files exist. Use force=True to regenerate.")
-        else:
-            # Process documents in batches
-            for i in range(0, len(md_files), self.batch_size):
-                batch = md_files[i : i + self.batch_size]
-                self.logger.info(
-                    f"Processing batch {i//self.batch_size + 1}/{(len(md_files)//self.batch_size) + 1}"
-                )
-                await self._process_document_batch(batch)
-
-        self.logger.info("Index generation complete, building/updating search index.")
-        self.build_search_index(clear_cache=clear_bm25_cache)
-
-    def generate(self, sections: List[str], mode: str = "extended") -> str:
-        # Get all markdown files
-        all_files = glob.glob(str(self.docs_dir / "[0-9]*.md")) + glob.glob(
-            str(self.docs_dir / "[0-9]*.xs.md")
-        )
-
-        # Extract base names without extensions
-        base_docs = {
-            Path(f).name.split(".")[0]
-            for f in all_files
-            if not Path(f).name.endswith(".q.md")
-        }
-
-        # Filter by sections if provided
-        if sections:
-            base_docs = {
-                doc
-                for doc in base_docs
-                if any(section.lower() in doc.lower() for section in sections)
-            }
-
-        # Get file paths based on mode
-        files = []
-        for doc in sorted(
-            base_docs,
-            key=lambda x: int(x.split("_")[0]) if x.split("_")[0].isdigit() else 999999,
-        ):
-            if mode == "condensed":
-                xs_file = self.docs_dir / f"{doc}.xs.md"
-                regular_file = self.docs_dir / f"{doc}.md"
-                files.append(str(xs_file if xs_file.exists() else regular_file))
-            else:
-                files.append(str(self.docs_dir / f"{doc}.md"))
-
-        # Read and format content
-        content = []
-        for file in files:
-            try:
-                with open(file, "r", encoding="utf-8") as f:
-                    fname = Path(file).name
-                    content.append(f"{'#'*20}\n# {fname}\n{'#'*20}\n\n{f.read()}")
-            except Exception as e:
-                self.logger.error(f"Error reading {file}: {str(e)}")
-
-        return "\n\n---\n\n".join(content) if content else ""
-
-    def search(self, query: str, top_k: int = 5) -> str:
-        if not self.bm25_index:
-            return "No search index available. Call build_search_index() first."
-
-        query_tokens = self.preprocess_text(query)
-        doc_scores = self.bm25_index.get_scores(query_tokens)
-
-        mean_score = np.mean(doc_scores)
-        std_score = np.std(doc_scores)
-        score_threshold = mean_score + (0.25 * std_score)
-
-        file_data = self._aggregate_search_scores(
-            doc_scores=doc_scores,
-            score_threshold=score_threshold,
-            query_tokens=query_tokens,
-        )
-
-        ranked_files = sorted(
-            file_data.items(),
-            key=lambda x: (
-                x[1]["code_match_score"] * 2.0
-                + x[1]["match_count"] * 1.5
-                + x[1]["total_score"]
-            ),
-            reverse=True,
-        )[:top_k]
-
-        results = []
-        for file, _ in ranked_files:
-            main_doc = str(file).replace(".q.md", ".md")
-            if os.path.exists(self.docs_dir / main_doc):
-                with open(self.docs_dir / main_doc, "r", encoding="utf-8") as f:
-                    only_file_name = main_doc.split("/")[-1]
-                    content = ["#" * 20, f"# {only_file_name}", "#" * 20, "", f.read()]
-                    results.append("\n".join(content))
-
-        return "\n\n---\n\n".join(results)
-
-    def _aggregate_search_scores(
-        self, doc_scores: List[float], score_threshold: float, query_tokens: List[str]
-    ) -> Dict:
-        file_data = {}
-
-        for idx, score in enumerate(doc_scores):
-            if score <= score_threshold:
-                continue
-
-            fact = self.tokenized_facts[idx]
-            file_path = self.document_map[fact]
-
-            if file_path not in file_data:
-                file_data[file_path] = {
-                    "total_score": 0,
-                    "match_count": 0,
-                    "code_match_score": 0,
-                    "matched_facts": [],
-                }
-
-            components = fact.split("|") if "|" in fact else [fact]
-
-            code_match_score = 0
-            if len(components) == 3:
-                code_ref = components[2].strip()
-                code_tokens = self.preprocess_text(code_ref)
-                code_match_score = len(set(query_tokens) & set(code_tokens)) / len(
-                    query_tokens
-                )
-
-            file_data[file_path]["total_score"] += score
-            file_data[file_path]["match_count"] += 1
-            file_data[file_path]["code_match_score"] = max(
-                file_data[file_path]["code_match_score"], code_match_score
-            )
-            file_data[file_path]["matched_facts"].append(fact)
-
-        return file_data
-
-    def refresh_index(self) -> None:
-        """Convenience method for a full rebuild."""
-        self.build_search_index(clear_cache=True)

crawl4ai/legacy/version_manager.py

@@ -1,29 +0,0 @@
-# version_manager.py
-from pathlib import Path
-from packaging import version
-from . import __version__
-
-
-class VersionManager:
-    def __init__(self):
-        self.home_dir = Path.home() / ".crawl4ai"
-        self.version_file = self.home_dir / "version.txt"
-
-    def get_installed_version(self):
-        """Get the version recorded in home directory"""
-        if not self.version_file.exists():
-            return None
-        try:
-            return version.parse(self.version_file.read_text().strip())
-        except:
-            return None
-
-    def update_version(self):
-        """Update the version file to current library version"""
-        self.version_file.write_text(__version__.__version__)
-
-    def needs_update(self):
-        """Check if database needs update based on version"""
-        installed = self.get_installed_version()
-        current = version.parse(__version__.__version__)
-        return installed is None or installed < current