Merge branch 'develop' into release/v0.8.0

O-series (o1, o3) and GPT-5 models only support temperature=1.
Setting litellm.drop_params=True auto-drops unsupported parameters
instead of throwing UnsupportedParamsError.

Fixes temperature=0.01 error for these models in LLM extraction.

.github/workflows/docker-release.yml

@@ -0,0 +1,100 @@
+name: Docker Release
+on:
+  release:
+    types: [published]
+  push:
+    tags:
+      - 'docker-rebuild-v*'  # Allow manual Docker rebuilds via tags
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Free up disk space
+        run: |
+          echo "=== Disk space before cleanup ==="
+          df -h
+
+          # Remove unnecessary tools and libraries (frees ~25GB)
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/share/swift
+
+          # Clean apt cache
+          sudo apt-get clean
+
+          echo "=== Disk space after cleanup ==="
+          df -h
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Extract version from release or tag
+        id: get_version
+        run: |
+          if [ "${{ github.event_name }}" == "release" ]; then
+            # Triggered by release event
+            VERSION="${{ github.event.release.tag_name }}"
+            VERSION=${VERSION#v}  # Remove 'v' prefix
+          else
+            # Triggered by docker-rebuild-v* tag
+            VERSION=${GITHUB_REF#refs/tags/docker-rebuild-v}
+          fi
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          echo "Building Docker images for version: $VERSION"
+
+      - name: Extract major and minor versions
+        id: versions
+        run: |
+          VERSION=${{ steps.get_version.outputs.VERSION }}
+          MAJOR=$(echo $VERSION | cut -d. -f1)
+          MINOR=$(echo $VERSION | cut -d. -f1-2)
+          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
+          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
+          echo "Semantic versions - Major: $MAJOR, Minor: $MINOR"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
+            unclecode/crawl4ai:latest
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Summary
+        run: |
+          echo "## 🐳 Docker Release Complete!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Published Images" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Platforms" >> $GITHUB_STEP_SUMMARY
+          echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY
+          echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🚀 Pull Command" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
+          echo "docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY

.github/workflows/docs/ARCHITECTURE.md

@@ -0,0 +1,917 @@
+# Workflow Architecture Documentation
+
+## Overview
+
+This document describes the technical architecture of the split release pipeline for Crawl4AI.
+
+---
+
+## Architecture Diagram
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                         Developer                                │
+│                              │                                   │
+│                              ▼                                   │
+│                    git tag v1.2.3                               │
+│                    git push --tags                              │
+└──────────────────────────────┬──────────────────────────────────┘
+                               │
+                               ▼
+┌─────────────────────────────────────────────────────────────────┐
+│                      GitHub Repository                           │
+│                                                                  │
+│  ┌────────────────────────────────────────────────────────┐   │
+│  │                  Tag Event: v1.2.3                      │   │
+│  └────────────────────────────────────────────────────────┘   │
+│                              │                                   │
+│                              ▼                                   │
+│  ┌────────────────────────────────────────────────────────┐   │
+│  │           release.yml (Release Pipeline)               │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 1. Extract Version                            │     │   │
+│  │  │    v1.2.3 → 1.2.3                            │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 2. Validate Version                           │     │   │
+│  │  │    Tag == __version__.py                      │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 3. Build Python Package                       │     │   │
+│  │  │    - Source dist (.tar.gz)                    │     │   │
+│  │  │    - Wheel (.whl)                             │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 4. Upload to PyPI                             │     │   │
+│  │  │    - Authenticate with token                  │     │   │
+│  │  │    - Upload dist/*                            │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 5. Create GitHub Release                      │     │   │
+│  │  │    - Tag: v1.2.3                              │     │   │
+│  │  │    - Body: Install instructions               │     │   │
+│  │  │    - Status: Published                        │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  └────────────────────────────────────────────────────────┘   │
+│                              │                                   │
+│                              ▼                                   │
+│  ┌────────────────────────────────────────────────────────┐   │
+│  │         Release Event: published (v1.2.3)              │   │
+│  └────────────────────────────────────────────────────────┘   │
+│                              │                                   │
+│                              ▼                                   │
+│  ┌────────────────────────────────────────────────────────┐   │
+│  │         docker-release.yml (Docker Pipeline)           │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 1. Extract Version from Release               │     │   │
+│  │  │    github.event.release.tag_name → 1.2.3     │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 2. Parse Semantic Versions                    │     │   │
+│  │  │    1.2.3 → Major: 1, Minor: 1.2              │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 3. Setup Multi-Arch Build                     │     │   │
+│  │  │    - Docker Buildx                            │     │   │
+│  │  │    - QEMU emulation                           │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 4. Authenticate Docker Hub                    │     │   │
+│  │  │    - Username: DOCKER_USERNAME                │     │   │
+│  │  │    - Token: DOCKER_TOKEN                      │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 5. Build Multi-Arch Images                    │     │   │
+│  │  │    ┌────────────────┬────────────────┐       │     │   │
+│  │  │    │  linux/amd64   │  linux/arm64   │       │     │   │
+│  │  │    └────────────────┴────────────────┘       │     │   │
+│  │  │    Cache: GitHub Actions (type=gha)          │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  │  ┌──────────────────────────────────────────────┐     │   │
+│  │  │ 6. Push to Docker Hub                         │     │   │
+│  │  │    Tags:                                      │     │   │
+│  │  │    - unclecode/crawl4ai:1.2.3                │     │   │
+│  │  │    - unclecode/crawl4ai:1.2                  │     │   │
+│  │  │    - unclecode/crawl4ai:1                    │     │   │
+│  │  │    - unclecode/crawl4ai:latest               │     │   │
+│  │  └──────────────────────────────────────────────┘     │   │
+│  └────────────────────────────────────────────────────────┘   │
+└─────────────────────────────────────────────────────────────────┘
+                               │
+                               ▼
+┌─────────────────────────────────────────────────────────────────┐
+│                     External Services                            │
+│                                                                  │
+│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐         │
+│  │    PyPI      │  │  Docker Hub  │  │   GitHub     │         │
+│  │              │  │              │  │              │         │
+│  │  crawl4ai    │  │ unclecode/   │  │  Releases    │         │
+│  │  1.2.3       │  │ crawl4ai     │  │  v1.2.3      │         │
+│  └──────────────┘  └──────────────┘  └──────────────┘         │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+---
+
+## Component Details
+
+### 1. Release Pipeline (release.yml)
+
+#### Purpose
+Fast publication of Python package and GitHub release.
+
+#### Input
+- **Trigger**: Git tag matching `v*` (excluding `test-v*`)
+- **Example**: `v1.2.3`
+
+#### Processing Stages
+
+##### Stage 1: Version Extraction
+```bash
+Input:  refs/tags/v1.2.3
+Output: VERSION=1.2.3
+```
+
+**Implementation**:
+```bash
+TAG_VERSION=${GITHUB_REF#refs/tags/v}  # Remove 'refs/tags/v' prefix
+echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
+```
+
+##### Stage 2: Version Validation
+```bash
+Input:  TAG_VERSION=1.2.3
+Check:  crawl4ai/__version__.py contains __version__ = "1.2.3"
+Output: Pass/Fail
+```
+
+**Implementation**:
+```bash
+PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
+if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
+  exit 1
+fi
+```
+
+##### Stage 3: Package Build
+```bash
+Input:  Source code + pyproject.toml
+Output: dist/crawl4ai-1.2.3.tar.gz
+        dist/crawl4ai-1.2.3-py3-none-any.whl
+```
+
+**Implementation**:
+```bash
+python -m build
+# Uses build backend defined in pyproject.toml
+```
+
+##### Stage 4: PyPI Upload
+```bash
+Input:  dist/*.{tar.gz,whl}
+Auth:   PYPI_TOKEN
+Output: Package published to PyPI
+```
+
+**Implementation**:
+```bash
+twine upload dist/*
+# Environment:
+#   TWINE_USERNAME: __token__
+#   TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+```
+
+##### Stage 5: GitHub Release Creation
+```bash
+Input:  Tag: v1.2.3
+        Body: Markdown content
+Output: Published GitHub release
+```
+
+**Implementation**:
+```yaml
+uses: softprops/action-gh-release@v2
+with:
+  tag_name: v1.2.3
+  name: Release v1.2.3
+  body: |
+    Installation instructions and changelog
+  draft: false
+  prerelease: false
+```
+
+#### Output
+- **PyPI Package**: https://pypi.org/project/crawl4ai/1.2.3/
+- **GitHub Release**: Published release on repository
+- **Event**: `release.published` (triggers Docker workflow)
+
+#### Timeline
+```
+0:00 - Tag pushed
+0:01 - Checkout + Python setup
+0:02 - Version validation
+0:03 - Package build
+0:04 - PyPI upload starts
+0:06 - PyPI upload complete
+0:07 - GitHub release created
+0:08 - Workflow complete
+```
+
+---
+
+### 2. Docker Release Pipeline (docker-release.yml)
+
+#### Purpose
+Build and publish multi-architecture Docker images.
+
+#### Inputs
+
+##### Input 1: Release Event (Automatic)
+```yaml
+Event: release.published
+Data:  github.event.release.tag_name = "v1.2.3"
+```
+
+##### Input 2: Docker Rebuild Tag (Manual)
+```yaml
+Tag: docker-rebuild-v1.2.3
+```
+
+#### Processing Stages
+
+##### Stage 1: Version Detection
+```bash
+# From release event:
+VERSION = github.event.release.tag_name.strip("v")
+# Result: "1.2.3"
+
+# From rebuild tag:
+VERSION = GITHUB_REF.replace("refs/tags/docker-rebuild-v", "")
+# Result: "1.2.3"
+```
+
+##### Stage 2: Semantic Version Parsing
+```bash
+Input:  VERSION=1.2.3
+Output: MAJOR=1
+        MINOR=1.2
+        PATCH=3 (implicit)
+```
+
+**Implementation**:
+```bash
+MAJOR=$(echo $VERSION | cut -d. -f1)    # Extract first component
+MINOR=$(echo $VERSION | cut -d. -f1-2)  # Extract first two components
+```
+
+##### Stage 3: Multi-Architecture Setup
+```yaml
+Setup:
+  - Docker Buildx (multi-platform builder)
+  - QEMU (ARM emulation on x86)
+
+Platforms:
+  - linux/amd64 (x86_64)
+  - linux/arm64 (aarch64)
+```
+
+**Architecture**:
+```
+GitHub Runner (linux/amd64)
+  ├─ Buildx Builder
+  │   ├─ Native: Build linux/amd64 image
+  │   └─ QEMU: Emulate ARM to build linux/arm64 image
+  └─ Generate manifest list (points to both images)
+```
+
+##### Stage 4: Docker Hub Authentication
+```bash
+Input:  DOCKER_USERNAME
+        DOCKER_TOKEN
+Output: Authenticated Docker client
+```
+
+##### Stage 5: Build with Cache
+```yaml
+Cache Configuration:
+  cache-from: type=gha           # Read from GitHub Actions cache
+  cache-to: type=gha,mode=max    # Write all layers
+
+Cache Key Components:
+  - Workflow file path
+  - Branch name
+  - Architecture (amd64/arm64)
+```
+
+**Cache Hierarchy**:
+```
+Cache Entry: main/docker-release.yml/linux-amd64
+  ├─ Layer: sha256:abc123... (FROM python:3.12)
+  ├─ Layer: sha256:def456... (RUN apt-get update)
+  ├─ Layer: sha256:ghi789... (COPY requirements.txt)
+  ├─ Layer: sha256:jkl012... (RUN pip install)
+  └─ Layer: sha256:mno345... (COPY . /app)
+
+Cache Hit/Miss Logic:
+  - If layer input unchanged → cache hit → skip build
+  - If layer input changed → cache miss → rebuild + all subsequent layers
+```
+
+##### Stage 6: Tag Generation
+```bash
+Input:  VERSION=1.2.3, MAJOR=1, MINOR=1.2
+
+Output Tags:
+  - unclecode/crawl4ai:1.2.3    (exact version)
+  - unclecode/crawl4ai:1.2      (minor version)
+  - unclecode/crawl4ai:1        (major version)
+  - unclecode/crawl4ai:latest   (latest stable)
+```
+
+**Tag Strategy**:
+- All tags point to same image SHA
+- Users can pin to desired stability level
+- Pushing new version updates `1`, `1.2`, and `latest` automatically
+
+##### Stage 7: Push to Registry
+```bash
+For each tag:
+  For each platform (amd64, arm64):
+    Push image to Docker Hub
+
+Create manifest list:
+  Manifest: unclecode/crawl4ai:1.2.3
+    ├─ linux/amd64: sha256:abc...
+    └─ linux/arm64: sha256:def...
+
+Docker CLI automatically selects correct platform on pull
+```
+
+#### Output
+- **Docker Images**: 4 tags × 2 platforms = 8 image variants + 4 manifests
+- **Docker Hub**: https://hub.docker.com/r/unclecode/crawl4ai/tags
+
+#### Timeline
+
+**Cold Cache (First Build)**:
+```
+0:00 - Release event received
+0:01 - Checkout + Buildx setup
+0:02 - Docker Hub auth
+0:03 - Start build (amd64)
+0:08 - Complete amd64 build
+0:09 - Start build (arm64)
+0:14 - Complete arm64 build
+0:15 - Generate manifests
+0:16 - Push all tags
+0:17 - Workflow complete
+```
+
+**Warm Cache (Code Change Only)**:
+```
+0:00 - Release event received
+0:01 - Checkout + Buildx setup
+0:02 - Docker Hub auth
+0:03 - Start build (amd64) - cache hit for layers 1-4
+0:04 - Complete amd64 build (only layer 5 rebuilt)
+0:05 - Start build (arm64) - cache hit for layers 1-4
+0:06 - Complete arm64 build (only layer 5 rebuilt)
+0:07 - Generate manifests
+0:08 - Push all tags
+0:09 - Workflow complete
+```
+
+---
+
+## Data Flow
+
+### Version Information Flow
+
+```
+Developer
+  │
+  ▼
+crawl4ai/__version__.py
+  __version__ = "1.2.3"
+  │
+  ├─► Git Tag
+  │     v1.2.3
+  │       │
+  │       ▼
+  │     release.yml
+  │       │
+  │       ├─► Validation
+  │       │     ✓ Match
+  │       │
+  │       ├─► PyPI Package
+  │       │     crawl4ai==1.2.3
+  │       │
+  │       └─► GitHub Release
+  │             v1.2.3
+  │               │
+  │               ▼
+  │           docker-release.yml
+  │               │
+  │               └─► Docker Tags
+  │                     1.2.3, 1.2, 1, latest
+  │
+  └─► Package Metadata
+        pyproject.toml
+          version = "1.2.3"
+```
+
+### Secrets Flow
+
+```
+GitHub Secrets (Encrypted at Rest)
+  │
+  ├─► PYPI_TOKEN
+  │     │
+  │     ▼
+  │   release.yml
+  │     │
+  │     ▼
+  │   TWINE_PASSWORD env var (masked in logs)
+  │     │
+  │     ▼
+  │   PyPI API (HTTPS)
+  │
+  ├─► DOCKER_USERNAME
+  │     │
+  │     ▼
+  │   docker-release.yml
+  │     │
+  │     ▼
+  │   docker/login-action (masked in logs)
+  │     │
+  │     ▼
+  │   Docker Hub API (HTTPS)
+  │
+  └─► DOCKER_TOKEN
+        │
+        ▼
+      docker-release.yml
+        │
+        ▼
+      docker/login-action (masked in logs)
+        │
+        ▼
+      Docker Hub API (HTTPS)
+```
+
+### Artifact Flow
+
+```
+Source Code
+  │
+  ├─► release.yml
+  │     │
+  │     ▼
+  │   python -m build
+  │     │
+  │     ├─► crawl4ai-1.2.3.tar.gz
+  │     │     │
+  │     │     ▼
+  │     │   PyPI Storage
+  │     │     │
+  │     │     ▼
+  │     │   pip install crawl4ai
+  │     │
+  │     └─► crawl4ai-1.2.3-py3-none-any.whl
+  │           │
+  │           ▼
+  │         PyPI Storage
+  │           │
+  │           ▼
+  │         pip install crawl4ai
+  │
+  └─► docker-release.yml
+        │
+        ▼
+      docker build
+        │
+        ├─► Image: linux/amd64
+        │     │
+        │     └─► Docker Hub
+        │           unclecode/crawl4ai:1.2.3-amd64
+        │
+        └─► Image: linux/arm64
+              │
+              └─► Docker Hub
+                    unclecode/crawl4ai:1.2.3-arm64
+```
+
+---
+
+## State Machines
+
+### Release Pipeline State Machine
+
+```
+┌─────────┐
+│  START  │
+└────┬────┘
+     │
+     ▼
+┌──────────────┐
+│ Extract      │
+│ Version      │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐      ┌─────────┐
+│ Validate     │─────►│ FAILED  │
+│ Version      │ No   │ (Exit 1)│
+└──────┬───────┘      └─────────┘
+       │ Yes
+       ▼
+┌──────────────┐
+│ Build        │
+│ Package      │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐      ┌─────────┐
+│ Upload       │─────►│ FAILED  │
+│ to PyPI      │ Error│ (Exit 1)│
+└──────┬───────┘      └─────────┘
+       │ Success
+       ▼
+┌──────────────┐
+│ Create       │
+│ GH Release   │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐
+│  SUCCESS     │
+│ (Emit Event) │
+└──────────────┘
+```
+
+### Docker Pipeline State Machine
+
+```
+┌─────────┐
+│  START  │
+│ (Event) │
+└────┬────┘
+     │
+     ▼
+┌──────────────┐
+│ Detect       │
+│ Version      │
+│ Source       │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐
+│ Parse        │
+│ Semantic     │
+│ Versions     │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐      ┌─────────┐
+│ Authenticate │─────►│ FAILED  │
+│ Docker Hub   │ Error│ (Exit 1)│
+└──────┬───────┘      └─────────┘
+       │ Success
+       ▼
+┌──────────────┐
+│ Build        │
+│ amd64        │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐      ┌─────────┐
+│ Build        │─────►│ FAILED  │
+│ arm64        │ Error│ (Exit 1)│
+└──────┬───────┘      └─────────┘
+       │ Success
+       ▼
+┌──────────────┐
+│ Push All     │
+│ Tags         │
+└──────┬───────┘
+       │
+       ▼
+┌──────────────┐
+│  SUCCESS     │
+└──────────────┘
+```
+
+---
+
+## Security Architecture
+
+### Threat Model
+
+#### Threats Mitigated
+
+1. **Secret Exposure**
+   - Mitigation: GitHub Actions secret masking
+   - Evidence: Secrets never appear in logs
+
+2. **Unauthorized Package Upload**
+   - Mitigation: Scoped PyPI tokens
+   - Evidence: Token limited to `crawl4ai` project
+
+3. **Man-in-the-Middle**
+   - Mitigation: HTTPS for all API calls
+   - Evidence: PyPI, Docker Hub, GitHub all use TLS
+
+4. **Supply Chain Tampering**
+   - Mitigation: Immutable artifacts, content checksums
+   - Evidence: PyPI stores SHA256, Docker uses content-addressable storage
+
+#### Trust Boundaries
+
+```
+┌─────────────────────────────────────────┐
+│         Trusted Zone                     │
+│  ┌────────────────────────────────┐    │
+│  │  GitHub Actions Runner         │    │
+│  │  - Ephemeral VM                │    │
+│  │  - Isolated environment        │    │
+│  │  - Access to secrets           │    │
+│  └────────────────────────────────┘    │
+│                │                         │
+│                │ HTTPS (TLS 1.2+)       │
+│                ▼                         │
+└─────────────────────────────────────────┘
+                 │
+    ┌────────────┼────────────┐
+    │            │            │
+    ▼            ▼            ▼
+┌────────┐  ┌─────────┐  ┌──────────┐
+│  PyPI  │  │  Docker │  │  GitHub  │
+│  API   │  │  Hub    │  │  API     │
+└────────┘  └─────────┘  └──────────┘
+ External     External     External
+  Service      Service      Service
+```
+
+### Secret Management
+
+#### Secret Lifecycle
+
+```
+Creation (Developer)
+  │
+  ├─► PyPI: Create API token (scoped to project)
+  ├─► Docker Hub: Create access token (read/write)
+  │
+  ▼
+Storage (GitHub)
+  │
+  ├─► Encrypted at rest (AES-256)
+  ├─► Access controlled (repo-scoped)
+  │
+  ▼
+Usage (Workflow)
+  │
+  ├─► Injected as env vars
+  ├─► Masked in logs (GitHub redacts on output)
+  ├─► Never persisted to disk (in-memory only)
+  │
+  ▼
+Transmission (API Call)
+  │
+  ├─► HTTPS only
+  ├─► TLS 1.2+ with strong ciphers
+  │
+  ▼
+Rotation (Manual)
+  │
+  └─► Regenerate on PyPI/Docker Hub
+      Update GitHub secret
+```
+
+---
+
+## Performance Characteristics
+
+### Release Pipeline Performance
+
+| Metric | Value | Notes |
+|--------|-------|-------|
+| Cold start | ~2-3 min | First run on new runner |
+| Warm start | ~2-3 min | Minimal caching benefit |
+| PyPI upload | ~30-60 sec | Network-bound |
+| Package build | ~30 sec | CPU-bound |
+| Parallelization | None | Sequential by design |
+
+### Docker Pipeline Performance
+
+| Metric | Cold Cache | Warm Cache (code) | Warm Cache (deps) |
+|--------|-----------|-------------------|-------------------|
+| Total time | 10-15 min | 1-2 min | 3-5 min |
+| amd64 build | 5-7 min | 30-60 sec | 1-2 min |
+| arm64 build | 5-7 min | 30-60 sec | 1-2 min |
+| Push time | 1-2 min | 30 sec | 30 sec |
+| Cache hit rate | 0% | 85% | 60% |
+
+### Cache Performance Model
+
+```python
+def estimate_build_time(changes):
+    base_time = 60  # seconds (setup + push)
+
+    if "Dockerfile" in changes:
+        return base_time + (10 * 60)  # Full rebuild: ~11 min
+    elif "requirements.txt" in changes:
+        return base_time + (3 * 60)   # Deps rebuild: ~4 min
+    elif any(f.endswith(".py") for f in changes):
+        return base_time + 60          # Code only: ~2 min
+    else:
+        return base_time               # No changes: ~1 min
+```
+
+---
+
+## Scalability Considerations
+
+### Current Limits
+
+| Resource | Limit | Impact |
+|----------|-------|--------|
+| Workflow concurrency | 20 (default) | Max 20 releases in parallel |
+| Artifact storage | 500 MB/artifact | PyPI packages small (<10 MB) |
+| Cache storage | 10 GB/repo | Docker layers fit comfortably |
+| Workflow run time | 6 hours | Plenty of headroom |
+
+### Scaling Strategies
+
+#### Horizontal Scaling (Multiple Repos)
+```
+crawl4ai (main)
+  ├─ release.yml
+  └─ docker-release.yml
+
+crawl4ai-plugins (separate)
+  ├─ release.yml
+  └─ docker-release.yml
+
+Each repo has independent:
+  - Secrets
+  - Cache (10 GB each)
+  - Concurrency limits (20 each)
+```
+
+#### Vertical Scaling (Larger Runners)
+```yaml
+jobs:
+  docker:
+    runs-on: ubuntu-latest-8-cores  # GitHub-hosted larger runner
+    # 4x faster builds for CPU-bound layers
+```
+
+---
+
+## Disaster Recovery
+
+### Failure Scenarios
+
+#### Scenario 1: Release Pipeline Fails
+
+**Failure Point**: PyPI upload fails (network error)
+
+**State**:
+- ✓ Version validated
+- ✓ Package built
+- ✗ PyPI upload
+- ✗ GitHub release
+
+**Recovery**:
+```bash
+# Manual upload
+twine upload dist/*
+
+# Retry workflow (re-run from GitHub Actions UI)
+```
+
+**Prevention**: Add retry logic to PyPI upload
+
+#### Scenario 2: Docker Pipeline Fails
+
+**Failure Point**: ARM build fails (dependency issue)
+
+**State**:
+- ✓ PyPI published
+- ✓ GitHub release created
+- ✓ amd64 image built
+- ✗ arm64 image build
+
+**Recovery**:
+```bash
+# Fix Dockerfile
+git commit -am "fix: ARM build dependency"
+
+# Trigger rebuild
+git tag docker-rebuild-v1.2.3
+git push origin docker-rebuild-v1.2.3
+```
+
+**Impact**: PyPI package available, only Docker ARM users affected
+
+#### Scenario 3: Partial Release
+
+**Failure Point**: GitHub release creation fails
+
+**State**:
+- ✓ PyPI published
+- ✗ GitHub release
+- ✗ Docker images
+
+**Recovery**:
+```bash
+# Create release manually
+gh release create v1.2.3 \
+  --title "Release v1.2.3" \
+  --notes "..."
+
+# This triggers docker-release.yml automatically
+```
+
+---
+
+## Monitoring and Observability
+
+### Metrics to Track
+
+#### Release Pipeline
+- Success rate (target: >99%)
+- Duration (target: <3 min)
+- PyPI upload time (target: <60 sec)
+
+#### Docker Pipeline
+- Success rate (target: >95%)
+- Duration (target: <15 min cold, <2 min warm)
+- Cache hit rate (target: >80% for code changes)
+
+### Alerting
+
+**Critical Alerts**:
+- Release pipeline failure (blocks release)
+- PyPI authentication failure (expired token)
+
+**Warning Alerts**:
+- Docker build >15 min (performance degradation)
+- Cache hit rate <50% (cache issue)
+
+### Logging
+
+**GitHub Actions Logs**:
+- Retention: 90 days
+- Downloadable: Yes
+- Searchable: Limited
+
+**Recommended External Logging**:
+```yaml
+- name: Send logs to external service
+  if: failure()
+  run: |
+    curl -X POST https://logs.example.com/api/v1/logs \
+      -H "Content-Type: application/json" \
+      -d "{\"workflow\": \"${{ github.workflow }}\", \"status\": \"failed\"}"
+```
+
+---
+
+## Future Enhancements
+
+### Planned Improvements
+
+1. **Automated Changelog Generation**
+   - Use conventional commits
+   - Generate CHANGELOG.md automatically
+
+2. **Pre-release Testing**
+   - Test builds on `test-v*` tags
+   - Upload to TestPyPI
+
+3. **Notification System**
+   - Slack/Discord notifications on release
+   - Email on failure
+
+4. **Performance Optimization**
+   - Parallel Docker builds (amd64 + arm64 simultaneously)
+   - Persistent runners for better caching
+
+5. **Enhanced Validation**
+   - Smoke tests after PyPI upload
+   - Container security scanning
+
+---
+
+## References
+
+- [GitHub Actions Architecture](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions)
+- [Docker Build Cache](https://docs.docker.com/build/cache/)
+- [PyPI API Documentation](https://warehouse.pypa.io/api-reference/)
+
+---
+
+**Last Updated**: 2025-01-21
+**Version**: 2.0

.github/workflows/docs/WORKFLOW_REFERENCE.md

@@ -0,0 +1,287 @@
+# Workflow Quick Reference
+
+## Quick Commands
+
+### Standard Release
+```bash
+# 1. Update version
+vim crawl4ai/__version__.py  # Set to "1.2.3"
+
+# 2. Commit and tag
+git add crawl4ai/__version__.py
+git commit -m "chore: bump version to 1.2.3"
+git tag v1.2.3
+git push origin main
+git push origin v1.2.3
+
+# 3. Monitor
+# - PyPI: ~2-3 minutes
+# - Docker: ~1-15 minutes
+```
+
+### Docker Rebuild Only
+```bash
+git tag docker-rebuild-v1.2.3
+git push origin docker-rebuild-v1.2.3
+```
+
+### Delete Tag (Undo Release)
+```bash
+# Local
+git tag -d v1.2.3
+
+# Remote
+git push --delete origin v1.2.3
+
+# GitHub Release
+gh release delete v1.2.3
+```
+
+---
+
+## Workflow Triggers
+
+### release.yml
+| Event | Pattern | Example |
+|-------|---------|---------|
+| Tag push | `v*` | `v1.2.3` |
+| Excludes | `test-v*` | `test-v1.2.3` |
+
+### docker-release.yml
+| Event | Pattern | Example |
+|-------|---------|---------|
+| Release published | `release.published` | Automatic |
+| Tag push | `docker-rebuild-v*` | `docker-rebuild-v1.2.3` |
+
+---
+
+## Environment Variables
+
+### release.yml
+| Variable | Source | Example |
+|----------|--------|---------|
+| `VERSION` | Git tag | `1.2.3` |
+| `TWINE_USERNAME` | Static | `__token__` |
+| `TWINE_PASSWORD` | Secret | `pypi-Ag...` |
+| `GITHUB_TOKEN` | Auto | `ghp_...` |
+
+### docker-release.yml
+| Variable | Source | Example |
+|----------|--------|---------|
+| `VERSION` | Release/Tag | `1.2.3` |
+| `MAJOR` | Computed | `1` |
+| `MINOR` | Computed | `1.2` |
+| `DOCKER_USERNAME` | Secret | `unclecode` |
+| `DOCKER_TOKEN` | Secret | `dckr_pat_...` |
+
+---
+
+## Docker Tags Generated
+
+| Version | Tags Created |
+|---------|-------------|
+| v1.0.0 | `1.0.0`, `1.0`, `1`, `latest` |
+| v1.1.0 | `1.1.0`, `1.1`, `1`, `latest` |
+| v1.2.3 | `1.2.3`, `1.2`, `1`, `latest` |
+| v2.0.0 | `2.0.0`, `2.0`, `2`, `latest` |
+
+---
+
+## Workflow Outputs
+
+### release.yml
+| Output | Location | Time |
+|--------|----------|------|
+| PyPI Package | https://pypi.org/project/crawl4ai/ | ~2-3 min |
+| GitHub Release | Repository → Releases | ~2-3 min |
+| Workflow Summary | Actions → Run → Summary | Immediate |
+
+### docker-release.yml
+| Output | Location | Time |
+|--------|----------|------|
+| Docker Images | https://hub.docker.com/r/unclecode/crawl4ai | ~1-15 min |
+| Workflow Summary | Actions → Run → Summary | Immediate |
+
+---
+
+## Common Issues
+
+| Issue | Solution |
+|-------|----------|
+| Version mismatch | Update `crawl4ai/__version__.py` to match tag |
+| PyPI 403 Forbidden | Check `PYPI_TOKEN` secret |
+| PyPI 400 File exists | Version already published, increment version |
+| Docker auth failed | Regenerate `DOCKER_TOKEN` |
+| Docker build timeout | Check Dockerfile, review build logs |
+| Cache not working | First build on branch always cold |
+
+---
+
+## Secrets Checklist
+
+- [ ] `PYPI_TOKEN` - PyPI API token (project or account scope)
+- [ ] `DOCKER_USERNAME` - Docker Hub username
+- [ ] `DOCKER_TOKEN` - Docker Hub access token (read/write)
+- [ ] `GITHUB_TOKEN` - Auto-provided (no action needed)
+
+---
+
+## Workflow Dependencies
+
+### release.yml Dependencies
+```yaml
+Python: 3.12
+Actions:
+  - actions/checkout@v4
+  - actions/setup-python@v5
+  - softprops/action-gh-release@v2
+PyPI Packages:
+  - build
+  - twine
+```
+
+### docker-release.yml Dependencies
+```yaml
+Actions:
+  - actions/checkout@v4
+  - docker/setup-buildx-action@v3
+  - docker/login-action@v3
+  - docker/build-push-action@v5
+Docker:
+  - Buildx
+  - QEMU (for multi-arch)
+```
+
+---
+
+## Cache Information
+
+### Type
+- GitHub Actions Cache (`type=gha`)
+
+### Storage
+- **Limit**: 10GB per repository
+- **Retention**: 7 days for unused entries
+- **Cleanup**: Automatic LRU eviction
+
+### Performance
+| Scenario | Cache Hit | Build Time |
+|----------|-----------|------------|
+| First build | 0% | 10-15 min |
+| Code change only | 85% | 1-2 min |
+| Dependency update | 60% | 3-5 min |
+| No changes | 100% | 30-60 sec |
+
+---
+
+## Build Platforms
+
+| Platform | Architecture | Devices |
+|----------|--------------|---------|
+| linux/amd64 | x86_64 | Intel/AMD servers, AWS EC2, GCP |
+| linux/arm64 | aarch64 | Apple Silicon, AWS Graviton, Raspberry Pi |
+
+---
+
+## Version Validation
+
+### Pre-Tag Checklist
+```bash
+# Check current version
+python -c "from crawl4ai.__version__ import __version__; print(__version__)"
+
+# Verify it matches intended tag
+# If tag is v1.2.3, version should be "1.2.3"
+```
+
+### Post-Release Verification
+```bash
+# PyPI
+pip install crawl4ai==1.2.3
+python -c "import crawl4ai; print(crawl4ai.__version__)"
+
+# Docker
+docker pull unclecode/crawl4ai:1.2.3
+docker run unclecode/crawl4ai:1.2.3 python -c "import crawl4ai; print(crawl4ai.__version__)"
+```
+
+---
+
+## Monitoring URLs
+
+| Service | URL |
+|---------|-----|
+| GitHub Actions | `https://github.com/{owner}/{repo}/actions` |
+| PyPI Project | `https://pypi.org/project/crawl4ai/` |
+| Docker Hub | `https://hub.docker.com/r/unclecode/crawl4ai` |
+| GitHub Releases | `https://github.com/{owner}/{repo}/releases` |
+
+---
+
+## Rollback Strategy
+
+### PyPI (Cannot Delete)
+```bash
+# Increment patch version
+git tag v1.2.4
+git push origin v1.2.4
+```
+
+### Docker (Can Overwrite)
+```bash
+# Rebuild with fix
+git tag docker-rebuild-v1.2.3
+git push origin docker-rebuild-v1.2.3
+```
+
+### GitHub Release
+```bash
+# Delete release
+gh release delete v1.2.3
+
+# Delete tag
+git push --delete origin v1.2.3
+```
+
+---
+
+## Status Badge Markdown
+
+```markdown
+[![Release Pipeline](https://github.com/{owner}/{repo}/actions/workflows/release.yml/badge.svg)](https://github.com/{owner}/{repo}/actions/workflows/release.yml)
+
+[![Docker Release](https://github.com/{owner}/{repo}/actions/workflows/docker-release.yml/badge.svg)](https://github.com/{owner}/{repo}/actions/workflows/docker-release.yml)
+```
+
+---
+
+## Timeline Example
+
+```
+0:00 - Push tag v1.2.3
+0:01 - release.yml starts
+0:02 - Version validation passes
+0:03 - Package built
+0:04 - PyPI upload starts
+0:06 - PyPI upload complete ✓
+0:07 - GitHub release created ✓
+0:08 - release.yml complete
+0:08 - docker-release.yml triggered
+0:10 - Docker build starts
+0:12 - amd64 image built (cache hit)
+0:14 - arm64 image built (cache hit)
+0:15 - Images pushed to Docker Hub ✓
+0:16 - docker-release.yml complete
+
+Total: ~16 minutes
+Critical path (PyPI + GitHub): ~8 minutes
+```
+
+---
+
+## Contact
+
+For workflow issues:
+1. Check Actions tab for logs
+2. Review this reference
+3. See [README.md](./README.md) for detailed docs

.github/workflows/release.yml

@@ -10,53 +10,53 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write  # Required for creating releases
-    
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
-      
+
       - name: Extract version from tag
         id: get_version
         run: |
           TAG_VERSION=${GITHUB_REF#refs/tags/v}
           echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
           echo "Releasing version: $TAG_VERSION"
-      
+
       - name: Install package dependencies
         run: |
           pip install -e .
-      
+
       - name: Check version consistency
         run: |
           TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
           PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
-          
+
           echo "Tag version: $TAG_VERSION"
           echo "Package version: $PACKAGE_VERSION"
-          
+
           if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
             echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
             echo "Please update crawl4ai/__version__.py to match the tag version"
             exit 1
           fi
           echo "✅ Version check passed: $TAG_VERSION"
-      
+
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip
           pip install build twine
-      
+
       - name: Build package
         run: python -m build
-      
+
       - name: Check package
         run: twine check dist/*
-      
+
       - name: Upload to PyPI
         env:
           TWINE_USERNAME: __token__
@@ -65,37 +65,7 @@ jobs:
           echo "📦 Uploading to PyPI..."
           twine upload dist/*
           echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/"
-      
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      
-      - name: Extract major and minor versions
-        id: versions
-        run: |
-          VERSION=${{ steps.get_version.outputs.VERSION }}
-          MAJOR=$(echo $VERSION | cut -d. -f1)
-          MINOR=$(echo $VERSION | cut -d. -f1-2)
-          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
-          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
-      
-      - name: Build and push Docker images
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
-            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
-            unclecode/crawl4ai:latest
-          platforms: linux/amd64,linux/arm64
-      
+
       - name: Create GitHub Release
         uses: softprops/action-gh-release@v2
         with:
@@ -103,26 +73,29 @@ jobs:
           name: Release v${{ steps.get_version.outputs.VERSION }}
           body: |
             ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released!
-            
+
             ### 📦 Installation
-            
+
             **PyPI:**
             ```bash
             pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}
             ```
-            
+
             **Docker:**
             ```bash
             docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
             docker pull unclecode/crawl4ai:latest
             ```
-            
+
+            **Note:** Docker images are being built and will be available shortly.
+            Check the [Docker Release workflow](https://github.com/${{ github.repository }}/actions/workflows/docker-release.yml) for build status.
+
             ### 📝 What's Changed
             See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
           draft: false
           prerelease: false
           token: ${{ secrets.GITHUB_TOKEN }}
-      
+
       - name: Summary
         run: |
           echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY
@@ -132,11 +105,9 @@ jobs:
           echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
           echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
           echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
-          echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
+          echo "Docker images are being built in a separate workflow." >> $GITHUB_STEP_SUMMARY
+          echo "Check: https://github.com/${{ github.repository }}/actions/workflows/docker-release.yml" >> $GITHUB_STEP_SUMMARY

.github/workflows/release.yml.backup

@@ -0,0 +1,142 @@
+name: Release Pipeline
+on:
+  push:
+    tags:
+      - 'v*'
+      - '!test-v*'  # Exclude test tags
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # Required for creating releases
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      
+      - name: Extract version from tag
+        id: get_version
+        run: |
+          TAG_VERSION=${GITHUB_REF#refs/tags/v}
+          echo "VERSION=$TAG_VERSION" >> $GITHUB_OUTPUT
+          echo "Releasing version: $TAG_VERSION"
+      
+      - name: Install package dependencies
+        run: |
+          pip install -e .
+      
+      - name: Check version consistency
+        run: |
+          TAG_VERSION=${{ steps.get_version.outputs.VERSION }}
+          PACKAGE_VERSION=$(python -c "from crawl4ai.__version__ import __version__; print(__version__)")
+          
+          echo "Tag version: $TAG_VERSION"
+          echo "Package version: $PACKAGE_VERSION"
+          
+          if [ "$TAG_VERSION" != "$PACKAGE_VERSION" ]; then
+            echo "❌ Version mismatch! Tag: $TAG_VERSION, Package: $PACKAGE_VERSION"
+            echo "Please update crawl4ai/__version__.py to match the tag version"
+            exit 1
+          fi
+          echo "✅ Version check passed: $TAG_VERSION"
+      
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+      
+      - name: Build package
+        run: python -m build
+      
+      - name: Check package
+        run: twine check dist/*
+      
+      - name: Upload to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          echo "📦 Uploading to PyPI..."
+          twine upload dist/*
+          echo "✅ Package uploaded to https://pypi.org/project/crawl4ai/"
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      
+      - name: Extract major and minor versions
+        id: versions
+        run: |
+          VERSION=${{ steps.get_version.outputs.VERSION }}
+          MAJOR=$(echo $VERSION | cut -d. -f1)
+          MINOR=$(echo $VERSION | cut -d. -f1-2)
+          echo "MAJOR=$MAJOR" >> $GITHUB_OUTPUT
+          echo "MINOR=$MINOR" >> $GITHUB_OUTPUT
+      
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}
+            unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}
+            unclecode/crawl4ai:latest
+          platforms: linux/amd64,linux/arm64
+      
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ steps.get_version.outputs.VERSION }}
+          name: Release v${{ steps.get_version.outputs.VERSION }}
+          body: |
+            ## 🎉 Crawl4AI v${{ steps.get_version.outputs.VERSION }} Released!
+            
+            ### 📦 Installation
+            
+            **PyPI:**
+            ```bash
+            pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}
+            ```
+            
+            **Docker:**
+            ```bash
+            docker pull unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}
+            docker pull unclecode/crawl4ai:latest
+            ```
+            
+            ### 📝 What's Changed
+            See [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/main/CHANGELOG.md) for details.
+          draft: false
+          prerelease: false
+          token: ${{ secrets.GITHUB_TOKEN }}
+      
+      - name: Summary
+        run: |
+          echo "## 🚀 Release Complete!" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📦 PyPI Package" >> $GITHUB_STEP_SUMMARY
+          echo "- Version: ${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- URL: https://pypi.org/project/crawl4ai/" >> $GITHUB_STEP_SUMMARY
+          echo "- Install: \`pip install crawl4ai==${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 🐳 Docker Images" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.get_version.outputs.VERSION }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MINOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:${{ steps.versions.outputs.MAJOR }}\`" >> $GITHUB_STEP_SUMMARY
+          echo "- \`unclecode/crawl4ai:latest\`" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### 📋 GitHub Release" >> $GITHUB_STEP_SUMMARY
+          echo "https://github.com/${{ github.repository }}/releases/tag/v${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -266,6 +266,9 @@ continue_config.json
 .llm.env
 .private/
 
+.claude/
+.context/
+
 CLAUDE_MONITOR.md
 CLAUDE.md
 
@@ -293,3 +296,4 @@ scripts/
 *.db
 *.rdb
 *.ldb
+MEMORY.md

CHANGELOG.md

@@ -5,6 +5,46 @@ All notable changes to Crawl4AI will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.0] - 2026-01-12
+
+### Security
+- **🔒 CRITICAL: Remote Code Execution Fix**: Removed `__import__` from hook allowed builtins
+  - Prevents arbitrary module imports in user-provided hook code
+  - Hooks now disabled by default via `CRAWL4AI_HOOKS_ENABLED` environment variable
+  - Credit: Neo by ProjectDiscovery
+- **🔒 HIGH: Local File Inclusion Fix**: Added URL scheme validation to Docker API endpoints
+  - Blocks `file://`, `javascript:`, `data:` URLs on `/execute_js`, `/screenshot`, `/pdf`, `/html`
+  - Only allows `http://`, `https://`, and `raw:` URLs
+  - Credit: Neo by ProjectDiscovery
+
+### Breaking Changes
+- **Docker API: Hooks disabled by default**: Set `CRAWL4AI_HOOKS_ENABLED=true` to enable
+- **Docker API: file:// URLs blocked**: Use Python library directly for local file processing
+
+### Added
+- **🚀 init_scripts for BrowserConfig**: Pre-page-load JavaScript injection for stealth evasions
+- **🔄 CDP Connection Improvements**: WebSocket URL support, proper cleanup, browser reuse
+- **💾 Crash Recovery for Deep Crawl**: `resume_state` and `on_state_change` for BFS/DFS/Best-First strategies
+- **📄 PDF/MHTML for raw:/file:// URLs**: Generate PDFs and MHTML from cached HTML content
+- **📸 Screenshots for raw:/file:// URLs**: Render cached HTML and capture screenshots
+- **🔗 base_url Parameter**: Proper URL resolution for raw: HTML processing
+- **⚡ Prefetch Mode**: Two-phase deep crawling with fast link extraction
+- **🔀 Enhanced Proxy Support**: Improved proxy rotation and sticky sessions
+- **🌐 HTTP Strategy Proxy Support**: Non-browser crawler now supports proxies
+- **🖥️ Browser Pipeline for raw:/file://**: New `process_in_browser` parameter
+- **📋 Smart TTL Cache for Sitemap Seeder**: `cache_ttl_hours` and `validate_sitemap_lastmod` parameters
+- **📚 Security Documentation**: Added SECURITY.md with vulnerability reporting guidelines
+
+### Fixed
+- **raw: URL Parsing**: Fixed truncation at `#` character (CSS color codes like `#eee`)
+- **Caching System**: Various improvements to cache validation and persistence
+
+### Documentation
+- Multi-sample schema generation section
+- URL seeder smart TTL cache parameters
+- v0.8.0 migration guide
+- Security policy and disclosure process
+
 ## [Unreleased]
 
 ### Added

Dockerfile

@@ -1,7 +1,7 @@
 FROM python:3.12-slim-bookworm AS build
 
 # C4ai version
-ARG C4AI_VER=0.7.0-r1
+ARG C4AI_VER=0.8.0
 ENV C4AI_VERSION=$C4AI_VER
 LABEL c4ai.version=$C4AI_VER
 
@@ -167,6 +167,11 @@ RUN mkdir -p /home/appuser/.cache/ms-playwright \
 
 RUN crawl4ai-doctor
 
+# Ensure all cache directories belong to appuser
+# This fixes permission issues with .cache/url_seeder and other runtime cache dirs
+RUN mkdir -p /home/appuser/.cache \
+    && chown -R appuser:appuser /home/appuser/.cache
+
 # Copy application code
 COPY deploy/docker/* ${APP_HOME}/
 

README.md

@@ -12,6 +12,16 @@
 [![Downloads](https://static.pepy.tech/badge/crawl4ai/month)](https://pepy.tech/project/crawl4ai)
 [![GitHub Sponsors](https://img.shields.io/github/sponsors/unclecode?style=flat&logo=GitHub-Sponsors&label=Sponsors&color=pink)](https://github.com/sponsors/unclecode)
 
+---
+#### 🚀 Crawl4AI Cloud API — Closed Beta (Launching Soon)
+Reliable, large-scale web extraction, now built to be _**drastically more cost-effective**_ than any of the existing solutions.
+
+👉 **Apply [here](https://forms.gle/E9MyPaNXACnAMaqG7) for early access**  
+_We’ll be onboarding in phases and working closely with early users.
+Limited slots._
+
+---
+
 <p align="center">
     <a href="https://x.com/crawl4ai">
       <img src="https://img.shields.io/badge/Follow%20on%20X-000000?style=for-the-badge&logo=x&logoColor=white" alt="Follow on X" />
@@ -27,11 +37,13 @@
 
 Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines. Fast, controllable, battle tested by a 50k+ star community.
 
-[✨ Check out latest update v0.7.4](#-recent-updates)
+[✨ Check out latest update v0.8.0](#-recent-updates)
 
-✨ New in v0.7.4: Revolutionary LLM Table Extraction with intelligent chunking, enhanced concurrency fixes, memory management refactor, and critical stability improvements. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.4.md)
+✨ **New in v0.8.0**: Crash Recovery & Prefetch Mode! Deep crawl crash recovery with `resume_state` and `on_state_change` callbacks for long-running crawls. New `prefetch=True` mode for 5-10x faster URL discovery. Critical security fixes for Docker API (hooks disabled by default, file:// URLs blocked). [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.8.0.md)
 
-✨ Recent v0.7.3: Undetected Browser Support, Multi-URL Configurations, Memory Monitoring, Enhanced Table Extraction, GitHub Sponsors. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.3.md)
+✨ Recent v0.7.8: Stability & Bug Fix Release! 11 bug fixes addressing Docker API issues, LLM extraction improvements, URL handling fixes, and dependency updates. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.8.md)
+
+✨ Previous v0.7.7: Complete Self-Hosting Platform with Real-time Monitoring! Enterprise-grade monitoring dashboard, comprehensive REST API, WebSocket streaming, and smart browser pool management. [Release notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.7.md)
 
 <details>
   <summary>🤓 <strong>My Personal Story</strong></summary>
@@ -177,7 +189,7 @@ No rate-limited APIs. No lock-in. Build and own your data pipeline with direct g
 - 📸 **Screenshots**: Capture page screenshots during crawling for debugging or analysis.
 - 📂 **Raw Data Crawling**: Directly process raw HTML (`raw:`) or local files (`file://`).
 - 🔗 **Comprehensive Link Extraction**: Extracts internal, external links, and embedded iframe content.
-- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior.
+- 🛠️ **Customizable Hooks**: Define hooks at every step to customize crawling behavior (supports both string and function-based APIs).
 - 💾 **Caching**: Cache data for improved speed and to avoid redundant fetches.
 - 📄 **Metadata Extraction**: Retrieve structured metadata from web pages.
 - 📡 **IFrame Content Extraction**: Seamless extraction from embedded iframe content.
@@ -294,6 +306,7 @@ pip install -e ".[all]"             # Install all optional features
 ### New Docker Features
 
 The new Docker implementation includes:
+- **Real-time Monitoring Dashboard** with live system metrics and browser pool visibility
 - **Browser pooling** with page pre-warming for faster response times
 - **Interactive playground** to test and generate request code
 - **MCP integration** for direct connection to AI tools like Claude Code
@@ -308,7 +321,8 @@ The new Docker implementation includes:
 docker pull unclecode/crawl4ai:latest
 docker run -d -p 11235:11235 --name crawl4ai --shm-size=1g unclecode/crawl4ai:latest
 
-# Visit the playground at http://localhost:11235/playground
+# Visit the monitoring dashboard at http://localhost:11235/dashboard
+# Or the playground at http://localhost:11235/playground
 ```
 
 ### Quick Test
@@ -337,7 +351,7 @@ else:
     result = requests.get(f"http://localhost:11235/task/{task_id}")
 ```
 
-For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, environment variables, and usage examples, see our [Docker Deployment Guide](https://docs.crawl4ai.com/basic/docker-deployment/).
+For more examples, see our [Docker Examples](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/docker_example.py). For advanced configuration, monitoring features, and production deployment, see our [Self-Hosting Guide](https://docs.crawl4ai.com/core/self-hosting/).
 
 </details>
 
@@ -542,8 +556,199 @@ async def test_news_crawl():
 
 </details>
 
+---
+
+> **💡 Tip:** Some websites may use **CAPTCHA** based verification mechanisms to prevent automated access. If your workflow encounters such challenges, you may optionally integrate a third-party CAPTCHA-handling service such as <strong>[CapSolver](https://www.capsolver.com/blog/Partners/crawl4ai-capsolver/?utm_source=crawl4ai&utm_medium=github_pr&utm_campaign=crawl4ai_integration)</strong>. They support reCAPTCHA v2/v3, Cloudflare Turnstile, Challenge, AWS WAF, and more. Please ensure that your usage complies with the target website’s terms of service and applicable laws.
+
 ## ✨ Recent Updates
 
+<details open>
+<summary><strong>Version 0.8.0 Release Highlights - Crash Recovery & Prefetch Mode</strong></summary>
+
+This release introduces crash recovery for deep crawls, a new prefetch mode for fast URL discovery, and critical security fixes for Docker deployments.
+
+- **🔄 Deep Crawl Crash Recovery**:
+  - `on_state_change` callback fires after each URL for real-time state persistence
+  - `resume_state` parameter to continue from a saved checkpoint
+  - JSON-serializable state for Redis/database storage
+  - Works with BFS, DFS, and Best-First strategies
+  ```python
+  from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+  strategy = BFSDeepCrawlStrategy(
+      max_depth=3,
+      resume_state=saved_state,  # Continue from checkpoint
+      on_state_change=save_to_redis,  # Called after each URL
+  )
+  ```
+
+- **⚡ Prefetch Mode for Fast URL Discovery**:
+  - `prefetch=True` skips markdown, extraction, and media processing
+  - 5-10x faster than full processing
+  - Perfect for two-phase crawling: discover first, process selectively
+  ```python
+  config = CrawlerRunConfig(prefetch=True)
+  result = await crawler.arun("https://example.com", config=config)
+  # Returns HTML and links only - no markdown generation
+  ```
+
+- **🔒 Security Fixes (Docker API)**:
+  - Hooks disabled by default (`CRAWL4AI_HOOKS_ENABLED=false`)
+  - `file://` URLs blocked on API endpoints to prevent LFI
+  - `__import__` removed from hook execution sandbox
+
+[Full v0.8.0 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.8.0.md)
+
+</details>
+
+<details>
+<summary><strong>Version 0.7.8 Release Highlights - Stability & Bug Fix Release</strong></summary>
+
+This release focuses on stability with 11 bug fixes addressing issues reported by the community. No new features, but significant improvements to reliability.
+
+- **🐳 Docker API Fixes**:
+  - Fixed `ContentRelevanceFilter` deserialization in deep crawl requests (#1642)
+  - Fixed `ProxyConfig` JSON serialization in `BrowserConfig.to_dict()` (#1629)
+  - Fixed `.cache` folder permissions in Docker image (#1638)
+
+- **🤖 LLM Extraction Improvements**:
+  - Configurable rate limiter backoff with new `LLMConfig` parameters (#1269):
+    ```python
+    from crawl4ai import LLMConfig
+
+    config = LLMConfig(
+        provider="openai/gpt-4o-mini",
+        backoff_base_delay=5,           # Wait 5s on first retry
+        backoff_max_attempts=5,          # Try up to 5 times
+        backoff_exponential_factor=3     # Multiply delay by 3 each attempt
+    )
+    ```
+  - HTML input format support for `LLMExtractionStrategy` (#1178):
+    ```python
+    from crawl4ai import LLMExtractionStrategy
+
+    strategy = LLMExtractionStrategy(
+        llm_config=config,
+        instruction="Extract table data",
+        input_format="html"  # Now supports: "html", "markdown", "fit_markdown"
+    )
+    ```
+  - Fixed raw HTML URL variable - extraction strategies now receive `"Raw HTML"` instead of HTML blob (#1116)
+
+- **🔗 URL Handling**:
+  - Fixed relative URL resolution after JavaScript redirects (#1268)
+  - Fixed import statement formatting in extracted code (#1181)
+
+- **📦 Dependency Updates**:
+  - Replaced deprecated PyPDF2 with pypdf (#1412)
+  - Pydantic v2 ConfigDict compatibility - no more deprecation warnings (#678)
+
+- **🧠 AdaptiveCrawler**:
+  - Fixed query expansion to actually use LLM instead of hardcoded mock data (#1621)
+
+[Full v0.7.8 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.8.md)
+
+</details>
+
+<details>
+<summary><strong>Version 0.7.7 Release Highlights - The Self-Hosting & Monitoring Update</strong></summary>
+
+- **📊 Real-time Monitoring Dashboard**: Interactive web UI with live system metrics and browser pool visibility
+  ```python
+  # Access the monitoring dashboard
+  # Visit: http://localhost:11235/dashboard
+
+  # Real-time metrics include:
+  # - System health (CPU, memory, network, uptime)
+  # - Active and completed request tracking
+  # - Browser pool management (permanent/hot/cold)
+  # - Janitor cleanup events
+  # - Error monitoring with full context
+  ```
+
+- **🔌 Comprehensive Monitor API**: Complete REST API for programmatic access to all monitoring data
+  ```python
+  import httpx
+
+  async with httpx.AsyncClient() as client:
+      # System health
+      health = await client.get("http://localhost:11235/monitor/health")
+
+      # Request tracking
+      requests = await client.get("http://localhost:11235/monitor/requests")
+
+      # Browser pool status
+      browsers = await client.get("http://localhost:11235/monitor/browsers")
+
+      # Endpoint statistics
+      stats = await client.get("http://localhost:11235/monitor/endpoints/stats")
+  ```
+
+- **⚡ WebSocket Streaming**: Real-time updates every 2 seconds for custom dashboards
+- **🔥 Smart Browser Pool**: 3-tier architecture (permanent/hot/cold) with automatic promotion and cleanup
+- **🧹 Janitor System**: Automatic resource management with event logging
+- **🎮 Control Actions**: Manual browser management (kill, restart, cleanup) via API
+- **📈 Production Metrics**: 6 critical metrics for operational excellence with Prometheus integration
+- **🐛 Critical Bug Fixes**:
+  - Fixed async LLM extraction blocking issue (#1055)
+  - Enhanced DFS deep crawl strategy (#1607)
+  - Fixed sitemap parsing in AsyncUrlSeeder (#1598)
+  - Resolved browser viewport configuration (#1495)
+  - Fixed CDP timing with exponential backoff (#1528)
+  - Security update for pyOpenSSL (>=25.3.0)
+
+[Full v0.7.7 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.7.md)
+
+</details>
+
+<details>
+<summary><strong>Version 0.7.5 Release Highlights - The Docker Hooks & Security Update</strong></summary>
+
+- **🔧 Docker Hooks System**: Complete pipeline customization with user-provided Python functions at 8 key points
+- **✨ Function-Based Hooks API (NEW)**: Write hooks as regular Python functions with full IDE support:
+  ```python
+  from crawl4ai import hooks_to_string
+  from crawl4ai.docker_client import Crawl4aiDockerClient
+
+  # Define hooks as regular Python functions
+  async def on_page_context_created(page, context, **kwargs):
+      """Block images to speed up crawling"""
+      await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+      await page.set_viewport_size({"width": 1920, "height": 1080})
+      return page
+
+  async def before_goto(page, context, url, **kwargs):
+      """Add custom headers"""
+      await page.set_extra_http_headers({'X-Crawl4AI': 'v0.7.5'})
+      return page
+
+  # Option 1: Use hooks_to_string() utility for REST API
+  hooks_code = hooks_to_string({
+      "on_page_context_created": on_page_context_created,
+      "before_goto": before_goto
+  })
+
+  # Option 2: Docker client with automatic conversion (Recommended)
+  client = Crawl4aiDockerClient(base_url="http://localhost:11235")
+  results = await client.crawl(
+      urls=["https://httpbin.org/html"],
+      hooks={
+          "on_page_context_created": on_page_context_created,
+          "before_goto": before_goto
+      }
+  )
+  # ✓ Full IDE support, type checking, and reusability!
+  ```
+
+- **🤖 Enhanced LLM Integration**: Custom providers with temperature control and base_url configuration
+- **🔒 HTTPS Preservation**: Secure internal link handling with `preserve_https_for_internal_links=True`
+- **🐍 Python 3.10+ Support**: Modern language features and enhanced performance
+- **🛠️ Bug Fixes**: Resolved multiple community-reported issues including URL processing, JWT authentication, and proxy configuration
+
+[Full v0.7.5 Release Notes →](https://github.com/unclecode/crawl4ai/blob/main/docs/blog/release-v0.7.5.md)
+
+</details>
+
 <details>
 <summary><strong>Version 0.7.4 Release Highlights - The Intelligent Table Extraction & Performance Update</strong></summary>
 
@@ -919,6 +1124,40 @@ We envision a future where AI is powered by real human knowledge, ensuring data
 For more details, see our [full mission statement](./MISSION.md).
 </details>
 
+## 🌟 Current Sponsors
+
+### 🏢 Enterprise Sponsors & Partners
+
+Our enterprise sponsors and technology partners help scale Crawl4AI to power production-grade data pipelines.
+
+| Company | About | Sponsorship Tier |
+|------|------|----------------------------|
+| <a href="https://app.nstproxy.com/register?i=ecOqW9" target="_blank"><picture><source width="250" media="(prefers-color-scheme: dark)" srcset="https://gist.github.com/aravindkarnam/62f82bd4818d3079d9dd3c31df432cf8/raw/nst-light.svg"><source width="250" media="(prefers-color-scheme: light)" srcset="https://www.nstproxy.com/logo.svg"><img alt="nstproxy" src="ttps://www.nstproxy.com/logo.svg"></picture></a>  | NstProxy is a trusted proxy provider with over 110M+ real residential IPs, city-level targeting, 99.99% uptime, and low pricing at $0.1/GB, it delivers unmatched stability, scale, and cost-efficiency. | 🥈 Silver |
+| <a href="https://app.scrapeless.com/passport/register?utm_source=official&utm_term=crawl4ai" target="_blank"><picture><source width="250" media="(prefers-color-scheme: dark)" srcset="https://gist.githubusercontent.com/aravindkarnam/0d275b942705604263e5c32d2db27bc1/raw/Scrapeless-light-logo.svg"><source width="250" media="(prefers-color-scheme: light)" srcset="https://gist.githubusercontent.com/aravindkarnam/22d0525cc0f3021bf19ebf6e11a69ccd/raw/Scrapeless-dark-logo.svg"><img alt="Scrapeless" src="https://gist.githubusercontent.com/aravindkarnam/22d0525cc0f3021bf19ebf6e11a69ccd/raw/Scrapeless-dark-logo.svg"></picture></a>  | Scrapeless provides production-grade infrastructure for Crawling, Automation, and AI Agents, offering Scraping Browser, 4 Proxy Types and Universal Scraping API. | 🥈 Silver |
+| <a href="https://dashboard.capsolver.com/passport/register?inviteCode=ESVSECTX5Q23" target="_blank"><picture><source width="120" media="(prefers-color-scheme: dark)" srcset="https://docs.crawl4ai.com/uploads/sponsors/20251013045338_72a71fa4ee4d2f40.png"><source width="120" media="(prefers-color-scheme: light)" srcset="https://www.capsolver.com/assets/images/logo-text.png"><img alt="Capsolver" src="https://www.capsolver.com/assets/images/logo-text.png"></picture></a> | AI-powered Captcha solving service. Supports all major Captcha types, including reCAPTCHA, Cloudflare, and more | 🥉 Bronze |
+| <a href="https://kipo.ai" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013045751_2d54f57f117c651e.png" alt="DataSync" width="120"/></a> | Helps engineers and buyers find, compare, and source electronic & industrial parts in seconds, with specs, pricing, lead times & alternatives.| 🥇 Gold |
+| <a href="https://www.kidocode.com/" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013045045_bb8dace3f0440d65.svg" alt="Kidocode" width="120"/><p align="center">KidoCode</p></a> | Kidocode is a hybrid technology and entrepreneurship school for kids aged 5–18, offering both online and on-campus education. | 🥇 Gold |
+| <a href="https://www.alephnull.sg/" target="_blank"><img src="https://docs.crawl4ai.com/uploads/sponsors/20251013050323_a9e8e8c4c3650421.svg" alt="Aleph null" width="120"/></a> | Singapore-based  Aleph Null is Asia’s leading edtech hub, dedicated to student-centric, AI-driven education—empowering learners with the tools to thrive in a fast-changing world. | 🥇 Gold |
+
+
+
+### 🧑‍🤝 Individual Sponsors
+
+A heartfelt thanks to our individual supporters! Every contribution helps us keep our opensource mission alive and thriving!
+
+<p align="left">
+  <a href="https://github.com/hafezparast"><img src="https://avatars.githubusercontent.com/u/14273305?s=60&v=4" style="border-radius:50%;" width="64px;"/></a>
+  <a href="https://github.com/ntohidi"><img src="https://avatars.githubusercontent.com/u/17140097?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
+  <a href="https://github.com/Sjoeborg"><img src="https://avatars.githubusercontent.com/u/17451310?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
+  <a href="https://github.com/romek-rozen"><img src="https://avatars.githubusercontent.com/u/30595969?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
+  <a href="https://github.com/Kourosh-Kiyani"><img src="https://avatars.githubusercontent.com/u/34105600?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
+  <a href="https://github.com/Etherdrake"><img src="https://avatars.githubusercontent.com/u/67021215?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
+  <a href="https://github.com/shaman247"><img src="https://avatars.githubusercontent.com/u/211010067?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
+  <a href="https://github.com/work-flow-manager"><img src="https://avatars.githubusercontent.com/u/217665461?s=60&v=4" style="border-radius:50%;"width="64px;"/></a>
+</p>
+
+> Want to join them? [Sponsor Crawl4AI →](https://github.com/sponsors/unclecode)
+
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=unclecode/crawl4ai&type=Date)](https://star-history.com/#unclecode/crawl4ai&Date)

SECURITY.md

@@ -0,0 +1,122 @@
+# Security Policy
+
+## Supported Versions
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 0.8.x   | :white_check_mark: |
+| 0.7.x   | :x: (upgrade recommended) |
+| < 0.7   | :x:                |
+
+## Reporting a Vulnerability
+
+We take security vulnerabilities seriously. If you discover a security issue, please report it responsibly.
+
+### How to Report
+
+**DO NOT** open a public GitHub issue for security vulnerabilities.
+
+Instead, please report via one of these methods:
+
+1. **GitHub Security Advisories (Preferred)**
+   - Go to [Security Advisories](https://github.com/unclecode/crawl4ai/security/advisories)
+   - Click "New draft security advisory"
+   - Fill in the details
+
+2. **Email**
+   - Send details to: security@crawl4ai.com
+   - Use subject: `[SECURITY] Brief description`
+   - Include:
+     - Description of the vulnerability
+     - Steps to reproduce
+     - Potential impact
+     - Any suggested fixes
+
+### What to Expect
+
+- **Acknowledgment**: Within 48 hours
+- **Initial Assessment**: Within 7 days
+- **Resolution Timeline**: Depends on severity
+  - Critical: 24-72 hours
+  - High: 7 days
+  - Medium: 30 days
+  - Low: 90 days
+
+### Disclosure Policy
+
+- We follow responsible disclosure practices
+- We will coordinate with you on disclosure timing
+- Credit will be given to reporters (unless anonymity is requested)
+- We may request CVE assignment for significant vulnerabilities
+
+## Security Best Practices for Users
+
+### Docker API Deployment
+
+If you're running the Crawl4AI Docker API in production:
+
+1. **Enable Authentication**
+   ```yaml
+   # config.yml
+   security:
+     enabled: true
+     jwt_enabled: true
+   ```
+   ```bash
+   # Set a strong secret key
+   export SECRET_KEY="your-secure-random-key-here"
+   ```
+
+2. **Hooks are Disabled by Default** (v0.8.0+)
+   - Only enable if you trust all API users
+   - Set `CRAWL4AI_HOOKS_ENABLED=true` only when necessary
+
+3. **Network Security**
+   - Run behind a reverse proxy (nginx, traefik)
+   - Use HTTPS in production
+   - Restrict access to trusted IPs if possible
+
+4. **Container Security**
+   - Run as non-root user (default in our container)
+   - Use read-only filesystem where possible
+   - Limit container resources
+
+### Library Usage
+
+When using Crawl4AI as a Python library:
+
+1. **Validate URLs** before crawling untrusted input
+2. **Sanitize extracted content** before using in other systems
+3. **Be cautious with hooks** - they execute arbitrary code
+
+## Known Security Issues
+
+### Fixed in v0.8.0
+
+| ID | Severity | Description | Fix |
+|----|----------|-------------|-----|
+| CVE-pending-1 | CRITICAL | RCE via hooks `__import__` | Removed from allowed builtins |
+| CVE-pending-2 | HIGH | LFI via `file://` URLs | URL scheme validation added |
+
+See [Security Advisory](https://github.com/unclecode/crawl4ai/security/advisories) for details.
+
+## Security Features
+
+### v0.8.0+
+
+- **URL Scheme Validation**: Blocks `file://`, `javascript:`, `data:` URLs on API
+- **Hooks Disabled by Default**: Opt-in via `CRAWL4AI_HOOKS_ENABLED=true`
+- **Restricted Hook Builtins**: No `__import__`, `eval`, `exec`, `open`
+- **JWT Authentication**: Optional but recommended for production
+- **Rate Limiting**: Configurable request limits
+- **Security Headers**: X-Frame-Options, CSP, HSTS when enabled
+
+## Acknowledgments
+
+We thank the following security researchers for responsibly disclosing vulnerabilities:
+
+- **[Neo by ProjectDiscovery](https://projectdiscovery.io/blog/introducing-neo)** - RCE and LFI vulnerabilities (December 2025)
+
+---
+
+*Last updated: January 2026*

crawl4ai/__init__.py

@@ -72,6 +72,8 @@ from .deep_crawling import (
     BestFirstCrawlingStrategy,
     DFSDeepCrawlStrategy,
     DeepCrawlDecorator,
+    ContentRelevanceFilter,
+    ContentTypeScorer,
 )
 # NEW: Import AsyncUrlSeeder
 from .async_url_seeder import AsyncUrlSeeder
@@ -103,7 +105,8 @@ from .browser_adapter import (
 
 from .utils import (
     start_colab_display_server,
-    setup_colab_environment
+    setup_colab_environment,
+    hooks_to_string
 )
 
 __all__ = [
@@ -183,6 +186,7 @@ __all__ = [
     "ProxyConfig",
     "start_colab_display_server",
     "setup_colab_environment",
+    "hooks_to_string",
     # C4A Script additions
     "c4a_compile",
     "c4a_validate", 

crawl4ai/__version__.py

@@ -1,7 +1,7 @@
 # crawl4ai/__version__.py
 
 # This is the version that will be used for stable releases
-__version__ = "0.7.4"
+__version__ = "0.8.0"
 
 # For nightly builds, this gets set during build process
 __nightly_version__ = None

crawl4ai/adaptive_crawler.py

@@ -728,18 +728,18 @@ class EmbeddingStrategy(CrawlStrategy):
         provider = llm_config_dict.get('provider', 'openai/gpt-4o-mini') if llm_config_dict else 'openai/gpt-4o-mini'
         api_token = llm_config_dict.get('api_token') if llm_config_dict else None
         
-        # response = perform_completion_with_backoff(
-        #     provider=provider,
-        #     prompt_with_variables=prompt,
-        #     api_token=api_token,
-        #     json_response=True
-        # )
+        response = perform_completion_with_backoff(
+            provider=provider,
+            prompt_with_variables=prompt,
+            api_token=api_token,
+            json_response=True
+        )
         
-        # variations = json.loads(response.choices[0].message.content)
+        variations = json.loads(response.choices[0].message.content)
         
         
         # # Mock data with more variations for split
-        variations ={'queries': ['what are the best vegetables to use in fried rice?', 'how do I make vegetable fried rice from scratch?', 'can you provide a quick recipe for vegetable fried rice?', 'what cooking techniques are essential for perfect fried rice with vegetables?', 'how to add flavor to vegetable fried rice?', 'are there any tips for making healthy fried rice with vegetables?']}
+        # variations ={'queries': ['what are the best vegetables to use in fried rice?', 'how do I make vegetable fried rice from scratch?', 'can you provide a quick recipe for vegetable fried rice?', 'what cooking techniques are essential for perfect fried rice with vegetables?', 'how to add flavor to vegetable fried rice?', 'are there any tips for making healthy fried rice with vegetables?']}
         
         
         # variations = {'queries': [

crawl4ai/async_configs.py

@@ -1,6 +1,7 @@
+import importlib
 import os
-from typing import Union
 import warnings
+import requests
 from .config import (
     DEFAULT_PROVIDER,
     DEFAULT_PROVIDER_API_KEY,
@@ -26,14 +27,14 @@ from .table_extraction import TableExtractionStrategy, DefaultTableExtraction
 from .cache_context import CacheMode
 from .proxy_strategy import ProxyRotationStrategy
 
-from typing import Union, List, Callable
 import inspect
-from typing import Any, Dict, Optional
+from typing import Any, Callable, Dict, List, Optional, Union
 from enum import Enum
 
 # Type alias for URL matching
 UrlMatcher = Union[str, Callable[[str], bool], List[Union[str, Callable[[str], bool]]]]
 
+
 class MatchMode(Enum):
     OR = "or"
     AND = "and"
@@ -41,8 +42,7 @@ class MatchMode(Enum):
 # from .proxy_strategy import ProxyConfig
 
 
-
-def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict:
+def to_serializable_dict(obj: Any, ignore_default_value : bool = False):
     """
     Recursively convert an object to a serializable dictionary using {type, params} structure
     for complex objects.
@@ -109,8 +109,6 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False) -> Dict:
         #             if value is not None:
         #                 current_values[attr_name] = to_serializable_dict(value)
 
-            
-        
         return {
             "type": obj.__class__.__name__,
             "params": current_values
@@ -136,12 +134,20 @@ def from_serializable_dict(data: Any) -> Any:
         if data["type"] == "dict" and "value" in data:
             return {k: from_serializable_dict(v) for k, v in data["value"].items()}
 
-        # Import from crawl4ai for class instances
-        import crawl4ai
-
-        if hasattr(crawl4ai, data["type"]):
-            cls = getattr(crawl4ai, data["type"])
+        cls = None
+        # If you are receiving an error while trying to convert a dict to an object:
+        # Either add a module to `modules_paths` list, or add the `data["type"]` to the crawl4ai __init__.py file
+        module_paths = ["crawl4ai"]
+        for module_path in module_paths:
+            try:
+                mod = importlib.import_module(module_path)
+                if hasattr(mod, data["type"]):
+                    cls = getattr(mod, data["type"])
+                    break
+            except (ImportError, AttributeError):
+                continue
 
+        if cls is not None:
             # Handle Enum
             if issubclass(cls, Enum):
                 return cls(data["params"])
@@ -367,6 +373,20 @@ class BrowserConfig:
         use_managed_browser (bool): Launch the browser using a managed approach (e.g., via CDP), allowing
                                     advanced manipulation. Default: False.
         cdp_url (str): URL for the Chrome DevTools Protocol (CDP) endpoint. Default: "ws://localhost:9222/devtools/browser/".
+        browser_context_id (str or None): Pre-existing CDP browser context ID to use. When provided along with
+                                          cdp_url, the crawler will reuse this context instead of creating a new one.
+                                          Useful for cloud browser services that pre-create isolated contexts.
+                                          Default: None.
+        target_id (str or None): Pre-existing CDP target ID (page) to use. When provided along with
+                                 browser_context_id, the crawler will reuse this target instead of creating
+                                 a new page. Default: None.
+        cdp_cleanup_on_close (bool): When True and using cdp_url, the close() method will still clean up
+                                     the local Playwright client resources. Useful for cloud/server scenarios
+                                     where you don't own the remote browser but need to prevent memory leaks
+                                     from accumulated Playwright instances. Default: False.
+        create_isolated_context (bool): When True and using cdp_url, forces creation of a new browser context
+                                        instead of reusing the default context. Essential for concurrent crawls
+                                        on the same browser to prevent navigation conflicts. Default: False.
         debugging_port (int): Port for the browser debugging protocol. Default: 9222.
         use_persistent_context (bool): Use a persistent browser context (like a persistent profile).
                                        Automatically sets use_managed_browser=True. Default: False.
@@ -421,6 +441,10 @@ class BrowserConfig:
         browser_mode: str = "dedicated",
         use_managed_browser: bool = False,
         cdp_url: str = None,
+        browser_context_id: str = None,
+        target_id: str = None,
+        cdp_cleanup_on_close: bool = False,
+        create_isolated_context: bool = False,
         use_persistent_context: bool = False,
         user_data_dir: str = None,
         chrome_channel: str = "chromium",
@@ -453,13 +477,18 @@ class BrowserConfig:
         debugging_port: int = 9222,
         host: str = "localhost",
         enable_stealth: bool = False,
+        init_scripts: List[str] = None,
     ):
         
         self.browser_type = browser_type
-        self.headless = headless 
+        self.headless = headless
         self.browser_mode = browser_mode
         self.use_managed_browser = use_managed_browser
         self.cdp_url = cdp_url
+        self.browser_context_id = browser_context_id
+        self.target_id = target_id
+        self.cdp_cleanup_on_close = cdp_cleanup_on_close
+        self.create_isolated_context = create_isolated_context
         self.use_persistent_context = use_persistent_context
         self.user_data_dir = user_data_dir
         self.chrome_channel = chrome_channel or self.browser_type or "chromium"
@@ -508,6 +537,7 @@ class BrowserConfig:
         self.debugging_port = debugging_port
         self.host = host
         self.enable_stealth = enable_stealth
+        self.init_scripts = init_scripts if init_scripts is not None else []
 
         fa_user_agenr_generator = ValidUAGenerator()
         if self.user_agent_mode == "random":
@@ -555,6 +585,10 @@ class BrowserConfig:
             browser_mode=kwargs.get("browser_mode", "dedicated"),
             use_managed_browser=kwargs.get("use_managed_browser", False),
             cdp_url=kwargs.get("cdp_url"),
+            browser_context_id=kwargs.get("browser_context_id"),
+            target_id=kwargs.get("target_id"),
+            cdp_cleanup_on_close=kwargs.get("cdp_cleanup_on_close", False),
+            create_isolated_context=kwargs.get("create_isolated_context", False),
             use_persistent_context=kwargs.get("use_persistent_context", False),
             user_data_dir=kwargs.get("user_data_dir"),
             chrome_channel=kwargs.get("chrome_channel", "chromium"),
@@ -583,6 +617,7 @@ class BrowserConfig:
             debugging_port=kwargs.get("debugging_port", 9222),
             host=kwargs.get("host", "localhost"),
             enable_stealth=kwargs.get("enable_stealth", False),
+            init_scripts=kwargs.get("init_scripts", []),
         )
 
     def to_dict(self):
@@ -592,12 +627,16 @@ class BrowserConfig:
             "browser_mode": self.browser_mode,
             "use_managed_browser": self.use_managed_browser,
             "cdp_url": self.cdp_url,
+            "browser_context_id": self.browser_context_id,
+            "target_id": self.target_id,
+            "cdp_cleanup_on_close": self.cdp_cleanup_on_close,
+            "create_isolated_context": self.create_isolated_context,
             "use_persistent_context": self.use_persistent_context,
             "user_data_dir": self.user_data_dir,
             "chrome_channel": self.chrome_channel,
             "channel": self.channel,
             "proxy": self.proxy,
-            "proxy_config": self.proxy_config,
+            "proxy_config": self.proxy_config.to_dict() if self.proxy_config else None,
             "viewport_width": self.viewport_width,
             "viewport_height": self.viewport_height,
             "accept_downloads": self.accept_downloads,
@@ -618,9 +657,10 @@ class BrowserConfig:
             "debugging_port": self.debugging_port,
             "host": self.host,
             "enable_stealth": self.enable_stealth,
+            "init_scripts": self.init_scripts,
         }
 
-                
+
         return result
 
     def clone(self, **kwargs):
@@ -649,6 +689,85 @@ class BrowserConfig:
             return config
         return BrowserConfig.from_kwargs(config)
 
+    def set_nstproxy(
+        self,
+        token: str,
+        channel_id: str,
+        country: str = "ANY",
+        state: str = "",
+        city: str = "",
+        protocol: str = "http",
+        session_duration: int = 10,
+    ):
+        """
+        Fetch a proxy from NSTProxy API and automatically assign it to proxy_config.
+
+        Get your NSTProxy token from: https://app.nstproxy.com/profile
+
+        Args:
+            token (str): NSTProxy API token.
+            channel_id (str): NSTProxy channel ID.
+            country (str, optional): Country code (default: "ANY").
+            state (str, optional): State code (default: "").
+            city (str, optional): City name (default: "").
+            protocol (str, optional): Proxy protocol ("http" or "socks5"). Defaults to "http".
+            session_duration (int, optional): Session duration in minutes (0 = rotate each request). Defaults to 10.
+
+        Raises:
+            ValueError: If the API response format is invalid.
+            PermissionError: If the API returns an error message.
+        """
+
+        # --- Validate input early ---
+        if not token or not channel_id:
+            raise ValueError("[NSTProxy] token and channel_id are required")
+
+        if protocol not in ("http", "socks5"):
+            raise ValueError(f"[NSTProxy] Invalid protocol: {protocol}")
+
+        # --- Build NSTProxy API URL ---
+        params = {
+            "fType": 2,
+            "count": 1,
+            "channelId": channel_id,
+            "country": country,
+            "protocol": protocol,
+            "sessionDuration": session_duration,
+            "token": token,
+        }
+        if state:
+            params["state"] = state
+        if city:
+            params["city"] = city
+
+        url = "https://api.nstproxy.com/api/v1/generate/apiproxies"
+
+        try:
+            response = requests.get(url, params=params, timeout=10)
+            response.raise_for_status()
+
+            data = response.json()
+
+            # --- Handle API error response ---
+            if isinstance(data, dict) and data.get("err"):
+                raise PermissionError(f"[NSTProxy] API Error: {data.get('msg', 'Unknown error')}")
+
+            if not isinstance(data, list) or not data:
+                raise ValueError("[NSTProxy] Invalid API response — expected a non-empty list")
+
+            proxy_info = data[0]
+
+            # --- Apply proxy config ---
+            self.proxy_config = ProxyConfig(
+                server=f"{protocol}://{proxy_info['ip']}:{proxy_info['port']}",
+                username=proxy_info["username"],
+                password=proxy_info["password"],
+            )
+
+        except Exception as e:
+            print(f"[NSTProxy] ❌ Failed to set proxy: {e}")
+            raise
+
 class VirtualScrollConfig:
     """Configuration for virtual scroll handling.
     
@@ -914,6 +1033,18 @@ class CrawlerRunConfig():
         proxy_config (ProxyConfig or dict or None): Detailed proxy configuration, e.g. {"server": "...", "username": "..."}.
                                      If None, no additional proxy config. Default: None.
 
+        # Sticky Proxy Session Parameters
+        proxy_session_id (str or None): When set, maintains the same proxy for all requests sharing this session ID.
+                                        The proxy is acquired on first request and reused for subsequent requests.
+                                        Session expires when explicitly released or crawler context is closed.
+                                        Default: None.
+        proxy_session_ttl (int or None): Time-to-live for sticky session in seconds.
+                                         After TTL expires, a new proxy is acquired on next request.
+                                         Default: None (session lasts until explicitly released or crawler closes).
+        proxy_session_auto_release (bool): If True, automatically release the proxy session after a batch operation.
+                                           Useful for arun_many() to clean up sessions automatically.
+                                           Default: False.
+
         # Browser Location and Identity Parameters
         locale (str or None): Locale to use for the browser context (e.g., "en-US").
                              Default: None.
@@ -942,6 +1073,15 @@ class CrawlerRunConfig():
         shared_data (dict or None): Shared data to be passed between hooks.
                                      Default: None.
 
+        # Cache Validation Parameters (Smart Cache)
+        check_cache_freshness (bool): If True, validates cached content freshness using HTTP
+                                      conditional requests (ETag/Last-Modified) and head fingerprinting
+                                      before returning cached results. Avoids full browser crawls when
+                                      content hasn't changed. Only applies when cache_mode allows reads.
+                                      Default: False.
+        cache_validation_timeout (float): Timeout in seconds for cache validation HTTP requests.
+                                          Default: 10.0.
+
         # Page Navigation and Timing Parameters
         wait_until (str): The condition to wait for when navigating, e.g. "domcontentloaded".
                           Default: "domcontentloaded".
@@ -1048,6 +1188,12 @@ class CrawlerRunConfig():
         # Connection Parameters
         stream (bool): If True, enables streaming of crawled URLs as they are processed when used with arun_many.
                       Default: False.
+        process_in_browser (bool): If True, forces raw:/file:// URLs to be processed through the browser
+                                   pipeline (enabling js_code, wait_for, scrolling, etc.). When False (default),
+                                   raw:/file:// URLs use a fast path that returns HTML directly without browser
+                                   interaction. This is automatically enabled when browser-requiring parameters
+                                   are detected (js_code, wait_for, screenshot, pdf, etc.).
+                                   Default: False.
 
         check_robots_txt (bool): Whether to check robots.txt rules before crawling. Default: False
                                  Default: False.
@@ -1093,6 +1239,10 @@ class CrawlerRunConfig():
         scraping_strategy: ContentScrapingStrategy = None,
         proxy_config: Union[ProxyConfig, dict, None] = None,
         proxy_rotation_strategy: Optional[ProxyRotationStrategy] = None,
+        # Sticky Proxy Session Parameters
+        proxy_session_id: Optional[str] = None,
+        proxy_session_ttl: Optional[int] = None,
+        proxy_session_auto_release: bool = False,
         # Browser Location and Identity Parameters
         locale: Optional[str] = None,
         timezone_id: Optional[str] = None,
@@ -1107,6 +1257,9 @@ class CrawlerRunConfig():
         no_cache_read: bool = False,
         no_cache_write: bool = False,
         shared_data: dict = None,
+        # Cache Validation Parameters (Smart Cache)
+        check_cache_freshness: bool = False,
+        cache_validation_timeout: float = 10.0,
         # Page Navigation and Timing Parameters
         wait_until: str = "domcontentloaded",
         page_timeout: int = PAGE_TIMEOUT,
@@ -1160,7 +1313,10 @@ class CrawlerRunConfig():
         # Connection Parameters
         method: str = "GET",
         stream: bool = False,
+        prefetch: bool = False,  # When True, return only HTML + links (skip heavy processing)
+        process_in_browser: bool = False,  # Force browser processing for raw:/file:// URLs
         url: str = None,
+        base_url: str = None,  # Base URL for markdown link resolution (used with raw: HTML)
         check_robots_txt: bool = False,
         user_agent: str = None,
         user_agent_mode: str = None,
@@ -1179,6 +1335,7 @@ class CrawlerRunConfig():
     ):
         # TODO: Planning to set properties dynamically based on the __init__ signature
         self.url = url
+        self.base_url = base_url  # Base URL for markdown link resolution
 
         # Content Processing Parameters
         self.word_count_threshold = word_count_threshold
@@ -1203,7 +1360,12 @@ class CrawlerRunConfig():
             self.proxy_config = ProxyConfig.from_string(proxy_config)
 
         self.proxy_rotation_strategy = proxy_rotation_strategy
-        
+
+        # Sticky Proxy Session Parameters
+        self.proxy_session_id = proxy_session_id
+        self.proxy_session_ttl = proxy_session_ttl
+        self.proxy_session_auto_release = proxy_session_auto_release
+
         # Browser Location and Identity Parameters
         self.locale = locale
         self.timezone_id = timezone_id
@@ -1220,6 +1382,9 @@ class CrawlerRunConfig():
         self.no_cache_read = no_cache_read
         self.no_cache_write = no_cache_write
         self.shared_data = shared_data
+        # Cache Validation (Smart Cache)
+        self.check_cache_freshness = check_cache_freshness
+        self.cache_validation_timeout = cache_validation_timeout
 
         # Page Navigation and Timing Parameters
         self.wait_until = wait_until
@@ -1286,6 +1451,8 @@ class CrawlerRunConfig():
 
         # Connection Parameters
         self.stream = stream
+        self.prefetch = prefetch  # Prefetch mode: return only HTML + links
+        self.process_in_browser = process_in_browser  # Force browser processing for raw:/file:// URLs
         self.method = method
 
         # Robots.txt Handling Parameters
@@ -1483,6 +1650,10 @@ class CrawlerRunConfig():
             scraping_strategy=kwargs.get("scraping_strategy"),
             proxy_config=kwargs.get("proxy_config"),
             proxy_rotation_strategy=kwargs.get("proxy_rotation_strategy"),
+            # Sticky Proxy Session Parameters
+            proxy_session_id=kwargs.get("proxy_session_id"),
+            proxy_session_ttl=kwargs.get("proxy_session_ttl"),
+            proxy_session_auto_release=kwargs.get("proxy_session_auto_release", False),
             # Browser Location and Identity Parameters
             locale=kwargs.get("locale", None),
             timezone_id=kwargs.get("timezone_id", None),
@@ -1558,6 +1729,8 @@ class CrawlerRunConfig():
             # Connection Parameters
             method=kwargs.get("method", "GET"),
             stream=kwargs.get("stream", False),
+            prefetch=kwargs.get("prefetch", False),
+            process_in_browser=kwargs.get("process_in_browser", False),
             check_robots_txt=kwargs.get("check_robots_txt", False),
             user_agent=kwargs.get("user_agent"),
             user_agent_mode=kwargs.get("user_agent_mode"),
@@ -1567,6 +1740,7 @@ class CrawlerRunConfig():
             # Link Extraction Parameters
             link_preview_config=kwargs.get("link_preview_config"),
             url=kwargs.get("url"),
+            base_url=kwargs.get("base_url"),
             # URL Matching Parameters
             url_matcher=kwargs.get("url_matcher"),
             match_mode=kwargs.get("match_mode", MatchMode.OR),
@@ -1606,6 +1780,9 @@ class CrawlerRunConfig():
             "scraping_strategy": self.scraping_strategy,
             "proxy_config": self.proxy_config,
             "proxy_rotation_strategy": self.proxy_rotation_strategy,
+            "proxy_session_id": self.proxy_session_id,
+            "proxy_session_ttl": self.proxy_session_ttl,
+            "proxy_session_auto_release": self.proxy_session_auto_release,
             "locale": self.locale,
             "timezone_id": self.timezone_id,
             "geolocation": self.geolocation,
@@ -1662,6 +1839,8 @@ class CrawlerRunConfig():
             "capture_console_messages": self.capture_console_messages,
             "method": self.method,
             "stream": self.stream,
+            "prefetch": self.prefetch,
+            "process_in_browser": self.process_in_browser,
             "check_robots_txt": self.check_robots_txt,
             "user_agent": self.user_agent,
             "user_agent_mode": self.user_agent_mode,
@@ -1712,7 +1891,10 @@ class LLMConfig:
         frequency_penalty: Optional[float] = None,
         presence_penalty: Optional[float] = None,
         stop: Optional[List[str]] = None,
-        n: Optional[int] = None,    
+        n: Optional[int] = None,
+        backoff_base_delay: Optional[int] = None,
+        backoff_max_attempts: Optional[int] = None,
+        backoff_exponential_factor: Optional[int] = None,
     ):
         """Configuaration class for LLM provider and API token."""
         self.provider = provider
@@ -1741,6 +1923,9 @@ class LLMConfig:
         self.presence_penalty = presence_penalty
         self.stop = stop
         self.n = n
+        self.backoff_base_delay = backoff_base_delay if backoff_base_delay is not None else 2
+        self.backoff_max_attempts = backoff_max_attempts if backoff_max_attempts is not None else 3
+        self.backoff_exponential_factor = backoff_exponential_factor if backoff_exponential_factor is not None else 2
 
     @staticmethod
     def from_kwargs(kwargs: dict) -> "LLMConfig":
@@ -1754,7 +1939,10 @@ class LLMConfig:
             frequency_penalty=kwargs.get("frequency_penalty"),
             presence_penalty=kwargs.get("presence_penalty"),
             stop=kwargs.get("stop"),
-            n=kwargs.get("n")
+            n=kwargs.get("n"),
+            backoff_base_delay=kwargs.get("backoff_base_delay"),
+            backoff_max_attempts=kwargs.get("backoff_max_attempts"),
+            backoff_exponential_factor=kwargs.get("backoff_exponential_factor")
         )
 
     def to_dict(self):
@@ -1768,7 +1956,10 @@ class LLMConfig:
             "frequency_penalty": self.frequency_penalty,
             "presence_penalty": self.presence_penalty,
             "stop": self.stop,
-            "n": self.n
+            "n": self.n,
+            "backoff_base_delay": self.backoff_base_delay,
+            "backoff_max_attempts": self.backoff_max_attempts,
+            "backoff_exponential_factor": self.backoff_exponential_factor
         }
 
     def clone(self, **kwargs):
@@ -1805,6 +1996,8 @@ class SeedingConfig:
         score_threshold: Optional[float] = None,
         scoring_method: str = "bm25",
         filter_nonsense_urls: bool = True,
+        cache_ttl_hours: int = 24,
+        validate_sitemap_lastmod: bool = True,
     ):
         """
         Initialize URL seeding configuration.
@@ -1836,10 +2029,14 @@ class SeedingConfig:
                   Requires extract_head=True. Default: None
             score_threshold: Minimum relevance score (0.0-1.0) to include URL. 
                            Only applies when query is provided. Default: None
-            scoring_method: Scoring algorithm to use. Currently only "bm25" is supported. 
+            scoring_method: Scoring algorithm to use. Currently only "bm25" is supported.
                           Future: "semantic". Default: "bm25"
-            filter_nonsense_urls: Filter out utility URLs like robots.txt, sitemap.xml, 
+            filter_nonsense_urls: Filter out utility URLs like robots.txt, sitemap.xml,
                                  ads.txt, favicon.ico, etc. Default: True
+            cache_ttl_hours: Hours before sitemap cache expires. Set to 0 to disable TTL
+                            (only lastmod validation). Default: 24
+            validate_sitemap_lastmod: If True, compares sitemap's <lastmod> with cache
+                                     timestamp and refetches if sitemap is newer. Default: True
         """
         self.source = source
         self.pattern = pattern
@@ -1856,6 +2053,8 @@ class SeedingConfig:
         self.score_threshold = score_threshold
         self.scoring_method = scoring_method
         self.filter_nonsense_urls = filter_nonsense_urls
+        self.cache_ttl_hours = cache_ttl_hours
+        self.validate_sitemap_lastmod = validate_sitemap_lastmod
 
     # Add to_dict, from_kwargs, and clone methods for consistency
     def to_dict(self) -> Dict[str, Any]:

crawl4ai/async_crawler_strategy.py

@@ -452,48 +452,48 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
         if url.startswith(("http://", "https://", "view-source:")):
             return await self._crawl_web(url, config)
 
-        elif url.startswith("file://"):
-            # initialize empty lists for console messages
-            captured_console = []
-            
-            # Process local file
-            local_file_path = url[7:]  # Remove 'file://' prefix
-            if not os.path.exists(local_file_path):
-                raise FileNotFoundError(f"Local file not found: {local_file_path}")
-            with open(local_file_path, "r", encoding="utf-8") as f:
-                html = f.read()
-            if config.screenshot:
-                screenshot_data = await self._generate_screenshot_from_html(html)
-            if config.capture_console_messages:
-                page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
-                captured_console = await self._capture_console_messages(page, url)
-
-            return AsyncCrawlResponse(
-                html=html,
-                response_headers=response_headers,
-                status_code=status_code,
-                screenshot=screenshot_data,
-                get_delayed_content=None,
-                console_messages=captured_console,
+        elif url.startswith("file://") or url.startswith("raw://") or url.startswith("raw:"):
+            # Check if browser processing is required for file:// or raw: URLs
+            needs_browser = (
+                config.process_in_browser or
+                config.screenshot or
+                config.pdf or
+                config.capture_mhtml or
+                config.js_code or
+                config.wait_for or
+                config.scan_full_page or
+                config.remove_overlay_elements or
+                config.simulate_user or
+                config.magic or
+                config.process_iframes or
+                config.capture_console_messages or
+                config.capture_network_requests
             )
 
-        ##### 
-        # Since both "raw:" and "raw://" start with "raw:", the first condition is always true for both, so "raw://" will be sliced as "//...", which is incorrect.
-        # Fix: Check for "raw://" first, then "raw:"
-        # Also, the prefix "raw://" is actually 6 characters long, not 7, so it should be sliced accordingly: url[6:]
-        #####
-        elif url.startswith("raw://") or url.startswith("raw:"):
-            # Process raw HTML content
-            # raw_html = url[4:] if url[:4] == "raw:" else url[7:]
-            raw_html = url[6:] if url.startswith("raw://") else url[4:]
-            html = raw_html
-            if config.screenshot:
-                screenshot_data = await self._generate_screenshot_from_html(html)
+            if needs_browser:
+                # Route through _crawl_web() for full browser pipeline
+                # _crawl_web() will detect file:// and raw: URLs and use set_content()
+                return await self._crawl_web(url, config)
+
+            # Fast path: return HTML directly without browser interaction
+            if url.startswith("file://"):
+                # Process local file
+                local_file_path = url[7:]  # Remove 'file://' prefix
+                if not os.path.exists(local_file_path):
+                    raise FileNotFoundError(f"Local file not found: {local_file_path}")
+                with open(local_file_path, "r", encoding="utf-8") as f:
+                    html = f.read()
+            else:
+                # Process raw HTML content (raw:// or raw:)
+                html = url[6:] if url.startswith("raw://") else url[4:]
+
             return AsyncCrawlResponse(
                 html=html,
                 response_headers=response_headers,
                 status_code=status_code,
-                screenshot=screenshot_data,
+                screenshot=None,
+                pdf_data=None,
+                mhtml_data=None,
                 get_delayed_content=None,
             )
         else:
@@ -666,67 +666,83 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
             if not config.js_only:
                 await self.execute_hook("before_goto", page, context=context, url=url, config=config)
 
-                try:
-                    # Generate a unique nonce for this request
-                    if config.experimental.get("use_csp_nonce", False):
-                        nonce = hashlib.sha256(os.urandom(32)).hexdigest()
+                # Check if this is a file:// or raw: URL that needs set_content() instead of goto()
+                is_local_content = url.startswith("file://") or url.startswith("raw://") or url.startswith("raw:")
 
-                        # Add CSP headers to the request
-                        await page.set_extra_http_headers(
-                            {
-                                "Content-Security-Policy": f"default-src 'self'; script-src 'self' 'nonce-{nonce}' 'strict-dynamic'"
-                            }
-                        )
-
-                    response = await page.goto(
-                        url, wait_until=config.wait_until, timeout=config.page_timeout
-                    )
-                    redirected_url = page.url
-                except Error as e:
-                    # Allow navigation to be aborted when downloading files
-                    # This is expected behavior for downloads in some browser engines
-                    if 'net::ERR_ABORTED' in str(e) and self.browser_config.accept_downloads:
-                        self.logger.info(
-                            message=f"Navigation aborted, likely due to file download: {url}",
-                            tag="GOTO",
-                            params={"url": url},
-                        )
-                        response = None
+                if is_local_content:
+                    # Load local content using set_content() instead of network navigation
+                    if url.startswith("file://"):
+                        local_file_path = url[7:]  # Remove 'file://' prefix
+                        if not os.path.exists(local_file_path):
+                            raise FileNotFoundError(f"Local file not found: {local_file_path}")
+                        with open(local_file_path, "r", encoding="utf-8") as f:
+                            html_content = f.read()
                     else:
-                        raise RuntimeError(f"Failed on navigating ACS-GOTO:\n{str(e)}")
+                        # raw:// or raw:
+                        html_content = url[6:] if url.startswith("raw://") else url[4:]
+
+                    await page.set_content(html_content, wait_until=config.wait_until)
+                    response = None
+                    redirected_url = config.base_url or url
+                    status_code = 200
+                    response_headers = {}
+                else:
+                    # Standard web navigation with goto()
+                    try:
+                        # Generate a unique nonce for this request
+                        if config.experimental.get("use_csp_nonce", False):
+                            nonce = hashlib.sha256(os.urandom(32)).hexdigest()
+
+                            # Add CSP headers to the request
+                            await page.set_extra_http_headers(
+                                {
+                                    "Content-Security-Policy": f"default-src 'self'; script-src 'self' 'nonce-{nonce}' 'strict-dynamic'"
+                                }
+                            )
+
+                        response = await page.goto(
+                            url, wait_until=config.wait_until, timeout=config.page_timeout
+                        )
+                        redirected_url = page.url
+                    except Error as e:
+                        # Allow navigation to be aborted when downloading files
+                        # This is expected behavior for downloads in some browser engines
+                        if 'net::ERR_ABORTED' in str(e) and self.browser_config.accept_downloads:
+                            self.logger.info(
+                                message=f"Navigation aborted, likely due to file download: {url}",
+                                tag="GOTO",
+                                params={"url": url},
+                            )
+                            response = None
+                        else:
+                            raise RuntimeError(f"Failed on navigating ACS-GOTO:\n{str(e)}")
+
+                    # ──────────────────────────────────────────────────────────────
+                    # Walk the redirect chain.  Playwright returns only the last
+                    # hop, so we trace the `request.redirected_from` links until the
+                    # first response that differs from the final one and surface its
+                    # status-code.
+                    # ──────────────────────────────────────────────────────────────
+                    if response is None:
+                        status_code = 200
+                        response_headers = {}
+                    else:
+                        first_resp = response
+                        req = response.request
+                        while req and req.redirected_from:
+                            prev_req = req.redirected_from
+                            prev_resp = await prev_req.response()
+                            if prev_resp:                       # keep earliest
+                                first_resp = prev_resp
+                            req = prev_req
+
+                        status_code = first_resp.status
+                        response_headers = first_resp.headers
 
                 await self.execute_hook(
                     "after_goto", page, context=context, url=url, response=response, config=config
                 )
 
-                # ──────────────────────────────────────────────────────────────
-                # Walk the redirect chain.  Playwright returns only the last
-                # hop, so we trace the `request.redirected_from` links until the
-                # first response that differs from the final one and surface its
-                # status-code.
-                # ──────────────────────────────────────────────────────────────
-                if response is None:
-                    status_code = 200
-                    response_headers = {}
-                else:
-                    first_resp = response
-                    req = response.request
-                    while req and req.redirected_from:
-                        prev_req = req.redirected_from
-                        prev_resp = await prev_req.response()
-                        if prev_resp:                       # keep earliest
-                            first_resp = prev_resp
-                        req = prev_req
-                
-                    status_code = first_resp.status
-                    response_headers = first_resp.headers
-                # if response is None:
-                #     status_code = 200
-                #     response_headers = {}
-                # else:
-                #     status_code = response.status
-                #     response_headers = response.headers
-
             else:
                 status_code = 200
                 response_headers = {}
@@ -1023,6 +1039,12 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
                 final_messages = await self.adapter.retrieve_console_messages(page)
                 captured_console.extend(final_messages)
 
+            ###
+            # This ensures we capture the current page URL at the time we return the response, 
+            # which correctly reflects any JavaScript navigation that occurred.
+            ###
+            redirected_url = page.url  # Use current page URL to capture JS redirects
+            
             # Return complete response
             return AsyncCrawlResponse(
                 html=html,
@@ -1383,9 +1405,10 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
         try:
             await self.adapter.evaluate(page,
                 f"""
-                (() => {{
+                (async () => {{
                     try {{
-                        {remove_overlays_js}
+                        const removeOverlays = {remove_overlays_js};
+                        await removeOverlays();
                         return {{ success: true }};
                     }} catch (error) {{
                         return {{
@@ -1517,7 +1540,78 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
         await page.goto(file_path)
 
         return captured_console
-        
+
+    async def _generate_media_from_html(
+        self, html: str, config: CrawlerRunConfig = None
+    ) -> tuple:
+        """
+        Generate media (screenshot, PDF, MHTML) from raw HTML content.
+
+        This method is used for raw: and file:// URLs where we have HTML content
+        but need to render it in a browser to generate media outputs.
+
+        Args:
+            html (str): The raw HTML content to render
+            config (CrawlerRunConfig, optional): Configuration for media options
+
+        Returns:
+            tuple: (screenshot_data, pdf_data, mhtml_data) - any can be None
+        """
+        page = None
+        screenshot_data = None
+        pdf_data = None
+        mhtml_data = None
+
+        try:
+            # Get a browser page
+            config = config or CrawlerRunConfig()
+            page, context = await self.browser_manager.get_page(crawlerRunConfig=config)
+
+            # Load the HTML content into the page
+            await page.set_content(html, wait_until="domcontentloaded")
+
+            # Generate requested media
+            if config.pdf:
+                pdf_data = await self.export_pdf(page)
+
+            if config.capture_mhtml:
+                mhtml_data = await self.capture_mhtml(page)
+
+            if config.screenshot:
+                if config.screenshot_wait_for:
+                    await asyncio.sleep(config.screenshot_wait_for)
+                screenshot_height_threshold = getattr(config, 'screenshot_height_threshold', None)
+                screenshot_data = await self.take_screenshot(
+                    page, screenshot_height_threshold=screenshot_height_threshold
+                )
+
+            return screenshot_data, pdf_data, mhtml_data
+
+        except Exception as e:
+            error_message = f"Failed to generate media from HTML: {str(e)}"
+            self.logger.error(
+                message="HTML media generation failed: {error}",
+                tag="ERROR",
+                params={"error": error_message},
+            )
+            # Return error image for screenshot if it was requested
+            if config and config.screenshot:
+                img = Image.new("RGB", (800, 600), color="black")
+                draw = ImageDraw.Draw(img)
+                font = ImageFont.load_default()
+                draw.text((10, 10), error_message, fill=(255, 255, 255), font=font)
+                buffered = BytesIO()
+                img.save(buffered, format="JPEG")
+                screenshot_data = base64.b64encode(buffered.getvalue()).decode("utf-8")
+            return screenshot_data, pdf_data, mhtml_data
+        finally:
+            # Clean up the page
+            if page:
+                try:
+                    await page.close()
+                except Exception:
+                    pass
+
     async def take_screenshot(self, page, **kwargs) -> str:
         """
         Take a screenshot of the current page.
@@ -2286,9 +2380,28 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
         )
 
 
+    def _format_proxy_url(self, proxy_config) -> str:
+        """Format ProxyConfig into aiohttp-compatible proxy URL."""
+        if not proxy_config:
+            return None
+
+        server = proxy_config.server
+        username = getattr(proxy_config, 'username', None)
+        password = getattr(proxy_config, 'password', None)
+
+        if username and password:
+            # Insert credentials into URL: http://user:pass@host:port
+            if '://' in server:
+                protocol, rest = server.split('://', 1)
+                return f"{protocol}://{username}:{password}@{rest}"
+            else:
+                return f"http://{username}:{password}@{server}"
+
+        return server
+
     async def _handle_http(
-        self, 
-        url: str, 
+        self,
+        url: str,
         config: CrawlerRunConfig
     ) -> AsyncCrawlResponse:
         async with self._session_context() as session:
@@ -2297,7 +2410,7 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
                 connect=10,
                 sock_read=30
             )
-            
+
             headers = dict(self._BASE_HEADERS)
             if self.browser_config.headers:
                 headers.update(self.browser_config.headers)
@@ -2309,6 +2422,12 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
                 'headers': headers
             }
 
+            # Add proxy support - use config.proxy_config (set by arun() from rotation strategy or direct config)
+            proxy_url = None
+            if config.proxy_config:
+                proxy_url = self._format_proxy_url(config.proxy_config)
+                request_kwargs['proxy'] = proxy_url
+
             if self.browser_config.method == "POST":
                 if self.browser_config.data:
                     request_kwargs['data'] = self.browser_config.data
@@ -2379,7 +2498,10 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
             if scheme == 'file':
                 return await self._handle_file(parsed.path)
             elif scheme == 'raw':
-                return await self._handle_raw(parsed.path)
+                # Don't use parsed.path - urlparse truncates at '#' which is common in CSS
+                # Strip prefix directly: "raw://" (6 chars) or "raw:" (4 chars)
+                raw_content = url[6:] if url.startswith("raw://") else url[4:]
+                return await self._handle_raw(raw_content)
             else:  # http or https
                 return await self._handle_http(url, config)
                 

crawl4ai/async_database.py

@@ -1,10 +1,11 @@
 import os
+import time
 from pathlib import Path
 import aiosqlite
 import asyncio
 from typing import Optional, Dict
 from contextlib import asynccontextmanager
-import json  
+import json
 from .models import CrawlResult, MarkdownGenerationResult, StringCompatibleMarkdown
 import aiofiles
 from .async_logger import AsyncLogger
@@ -262,6 +263,11 @@ class AsyncDatabaseManager:
                 "screenshot",
                 "response_headers",
                 "downloaded_files",
+                # Smart cache validation columns (added in 0.8.x)
+                "etag",
+                "last_modified",
+                "head_fingerprint",
+                "cached_at",
             ]
 
             for column in new_columns:
@@ -275,6 +281,11 @@ class AsyncDatabaseManager:
             await db.execute(
                 f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT "{{}}"'
             )
+        elif new_column == "cached_at":
+            # Timestamp column for cache validation
+            await db.execute(
+                f"ALTER TABLE crawled_data ADD COLUMN {new_column} REAL DEFAULT 0"
+            )
         else:
             await db.execute(
                 f'ALTER TABLE crawled_data ADD COLUMN {new_column} TEXT DEFAULT ""'
@@ -378,6 +389,92 @@ class AsyncDatabaseManager:
             )
             return None
 
+    async def aget_cache_metadata(self, url: str) -> Optional[Dict]:
+        """
+        Retrieve only cache validation metadata for a URL (lightweight query).
+
+        Returns dict with: url, etag, last_modified, head_fingerprint, cached_at, response_headers
+        This is used for cache validation without loading full content.
+        """
+        async def _get_metadata(db):
+            async with db.execute(
+                """SELECT url, etag, last_modified, head_fingerprint, cached_at, response_headers
+                   FROM crawled_data WHERE url = ?""",
+                (url,)
+            ) as cursor:
+                row = await cursor.fetchone()
+                if not row:
+                    return None
+
+                columns = [description[0] for description in cursor.description]
+                row_dict = dict(zip(columns, row))
+
+                # Parse response_headers JSON
+                try:
+                    row_dict["response_headers"] = (
+                        json.loads(row_dict["response_headers"])
+                        if row_dict["response_headers"] else {}
+                    )
+                except json.JSONDecodeError:
+                    row_dict["response_headers"] = {}
+
+                return row_dict
+
+        try:
+            return await self.execute_with_retry(_get_metadata)
+        except Exception as e:
+            self.logger.error(
+                message="Error retrieving cache metadata: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)},
+            )
+            return None
+
+    async def aupdate_cache_metadata(
+        self,
+        url: str,
+        etag: Optional[str] = None,
+        last_modified: Optional[str] = None,
+        head_fingerprint: Optional[str] = None,
+    ):
+        """
+        Update only the cache validation metadata for a URL.
+        Used to update etag/last_modified after a successful validation.
+        """
+        async def _update(db):
+            updates = []
+            values = []
+
+            if etag is not None:
+                updates.append("etag = ?")
+                values.append(etag)
+            if last_modified is not None:
+                updates.append("last_modified = ?")
+                values.append(last_modified)
+            if head_fingerprint is not None:
+                updates.append("head_fingerprint = ?")
+                values.append(head_fingerprint)
+
+            if not updates:
+                return
+
+            values.append(url)
+            await db.execute(
+                f"UPDATE crawled_data SET {', '.join(updates)} WHERE url = ?",
+                tuple(values)
+            )
+
+        try:
+            await self.execute_with_retry(_update)
+        except Exception as e:
+            self.logger.error(
+                message="Error updating cache metadata: {error}",
+                tag="ERROR",
+                force_verbose=True,
+                params={"error": str(e)},
+            )
+
     async def acache_url(self, result: CrawlResult):
         """Cache CrawlResult data"""
         # Store content files and get hashes
@@ -425,15 +522,24 @@ class AsyncDatabaseManager:
         for field, (content, content_type) in content_map.items():
             content_hashes[field] = await self._store_content(content, content_type)
 
+        # Extract cache validation headers from response
+        response_headers = result.response_headers or {}
+        etag = response_headers.get("etag") or response_headers.get("ETag") or ""
+        last_modified = response_headers.get("last-modified") or response_headers.get("Last-Modified") or ""
+        # head_fingerprint is set by caller via result attribute (if available)
+        head_fingerprint = getattr(result, "head_fingerprint", None) or ""
+        cached_at = time.time()
+
         async def _cache(db):
             await db.execute(
                 """
                 INSERT INTO crawled_data (
                     url, html, cleaned_html, markdown,
                     extracted_content, success, media, links, metadata,
-                    screenshot, response_headers, downloaded_files
+                    screenshot, response_headers, downloaded_files,
+                    etag, last_modified, head_fingerprint, cached_at
                 )
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 ON CONFLICT(url) DO UPDATE SET
                     html = excluded.html,
                     cleaned_html = excluded.cleaned_html,
@@ -445,7 +551,11 @@ class AsyncDatabaseManager:
                     metadata = excluded.metadata,
                     screenshot = excluded.screenshot,
                     response_headers = excluded.response_headers,
-                    downloaded_files = excluded.downloaded_files
+                    downloaded_files = excluded.downloaded_files,
+                    etag = excluded.etag,
+                    last_modified = excluded.last_modified,
+                    head_fingerprint = excluded.head_fingerprint,
+                    cached_at = excluded.cached_at
             """,
                 (
                     result.url,
@@ -460,6 +570,10 @@ class AsyncDatabaseManager:
                     content_hashes["screenshot"],
                     json.dumps(result.response_headers or {}),
                     json.dumps(result.downloaded_files or []),
+                    etag,
+                    last_modified,
+                    head_fingerprint,
+                    cached_at,
                 ),
             )
 

crawl4ai/async_url_seeder.py

@@ -24,7 +24,7 @@ import os
 import pathlib
 import re
 import time
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Sequence, Union
 from urllib.parse import quote, urljoin
@@ -78,6 +78,103 @@ _link_rx = re.compile(
 # ────────────────────────────────────────────────────────────────────────── helpers
 
 
+def _parse_sitemap_lastmod(xml_content: bytes) -> Optional[str]:
+    """Extract the most recent lastmod from sitemap XML."""
+    try:
+        if LXML:
+            root = etree.fromstring(xml_content)
+            # Get all lastmod elements (namespace-agnostic)
+            lastmods = root.xpath("//*[local-name()='lastmod']/text()")
+            if lastmods:
+                # Return the most recent one
+                return max(lastmods)
+    except Exception:
+        pass
+    return None
+
+
+def _is_cache_valid(
+    cache_path: pathlib.Path,
+    ttl_hours: int,
+    validate_lastmod: bool,
+    current_lastmod: Optional[str] = None
+) -> bool:
+    """
+    Check if sitemap cache is still valid.
+
+    Returns False (invalid) if:
+    - File doesn't exist
+    - File is corrupted/unreadable
+    - TTL expired (if ttl_hours > 0)
+    - Sitemap lastmod is newer than cache (if validate_lastmod=True)
+    """
+    if not cache_path.exists():
+        return False
+
+    try:
+        with open(cache_path, "r") as f:
+            data = json.load(f)
+
+        # Check version
+        if data.get("version") != 1:
+            return False
+
+        # Check TTL
+        if ttl_hours > 0:
+            created_at = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00"))
+            age_hours = (datetime.now(timezone.utc) - created_at).total_seconds() / 3600
+            if age_hours > ttl_hours:
+                return False
+
+        # Check lastmod
+        if validate_lastmod and current_lastmod:
+            cached_lastmod = data.get("sitemap_lastmod")
+            if cached_lastmod and current_lastmod > cached_lastmod:
+                return False
+
+        # Check URL count (sanity check - if 0, likely corrupted)
+        if data.get("url_count", 0) == 0:
+            return False
+
+        return True
+
+    except (json.JSONDecodeError, KeyError, ValueError, IOError):
+        # Corrupted cache - return False to trigger refetch
+        return False
+
+
+def _read_cache(cache_path: pathlib.Path) -> List[str]:
+    """Read URLs from cache file. Returns empty list on error."""
+    try:
+        with open(cache_path, "r") as f:
+            data = json.load(f)
+        return data.get("urls", [])
+    except Exception:
+        return []
+
+
+def _write_cache(
+    cache_path: pathlib.Path,
+    urls: List[str],
+    sitemap_url: str,
+    sitemap_lastmod: Optional[str]
+) -> None:
+    """Write URLs to cache with metadata."""
+    data = {
+        "version": 1,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "sitemap_lastmod": sitemap_lastmod,
+        "sitemap_url": sitemap_url,
+        "url_count": len(urls),
+        "urls": urls
+    }
+    try:
+        with open(cache_path, "w") as f:
+            json.dump(data, f)
+    except Exception:
+        pass  # Fail silently - cache is optional
+
+
 def _match(url: str, pattern: str) -> bool:
     if fnmatch.fnmatch(url, pattern):
         return True
@@ -295,6 +392,10 @@ class AsyncUrlSeeder:
         score_threshold = config.score_threshold
         scoring_method = config.scoring_method
 
+        # Store cache config for use in _from_sitemaps
+        self._cache_ttl_hours = getattr(config, 'cache_ttl_hours', 24)
+        self._validate_sitemap_lastmod = getattr(config, 'validate_sitemap_lastmod', True)
+
         # Ensure seeder's logger verbose matches the config's verbose if it's set
         if self.logger and hasattr(self.logger, 'verbose') and config.verbose is not None:
             self.logger.verbose = config.verbose
@@ -764,68 +865,222 @@ class AsyncUrlSeeder:
     # ─────────────────────────────── Sitemaps
     async def _from_sitemaps(self, domain: str, pattern: str, force: bool = False):
         """
-        1. Probe default sitemap locations.
-        2. If none exist, parse robots.txt for alternative sitemap URLs.
-        3. Yield only URLs that match `pattern`.
+        Discover URLs from sitemaps with smart TTL-based caching.
+
+        1. Check cache validity (TTL + lastmod)
+        2. If valid, yield from cache
+        3. If invalid or force=True, fetch fresh and update cache
+        4. FALLBACK: If anything fails, bypass cache and fetch directly
         """
+        # Get config values (passed via self during urls() call)
+        cache_ttl_hours = getattr(self, '_cache_ttl_hours', 24)
+        validate_lastmod = getattr(self, '_validate_sitemap_lastmod', True)
 
-       # ── cache file (same logic as _from_cc)
+        # Cache file path (new format: .json instead of .jsonl)
         host = re.sub(r'^https?://', '', domain).rstrip('/')
-        host = re.sub('[/?#]+', '_', domain)
+        host_safe = re.sub('[/?#]+', '_', host)
         digest = hashlib.md5(pattern.encode()).hexdigest()[:8]
-        path = self.cache_dir / f"sitemap_{host}_{digest}.jsonl"
+        cache_path = self.cache_dir / f"sitemap_{host_safe}_{digest}.json"
 
-        if path.exists() and not force:
-            self._log("info", "Loading sitemap URLs for {d} from cache: {p}",
-                      params={"d": host, "p": str(path)}, tag="URL_SEED")
-            async with aiofiles.open(path, "r") as fp:
-                async for line in fp:
-                    url = line.strip()
-                    if _match(url, pattern):
-                        yield url
-            return
+        # Check for old .jsonl format and delete it
+        old_cache_path = self.cache_dir / f"sitemap_{host_safe}_{digest}.jsonl"
+        if old_cache_path.exists():
+            try:
+                old_cache_path.unlink()
+                self._log("info", "Deleted old cache format: {p}",
+                          params={"p": str(old_cache_path)}, tag="URL_SEED")
+            except Exception:
+                pass
 
-        # 1️⃣ direct sitemap probe
-        # strip any scheme so we can handle https → http fallback
-        host = re.sub(r'^https?://', '', domain).rstrip('/')
+        # Step 1: Find sitemap URL and get lastmod (needed for validation)
+        sitemap_url = None
+        sitemap_lastmod = None
+        sitemap_content = None
 
-        schemes = ('https', 'http')  # prefer TLS, downgrade if needed
+        schemes = ('https', 'http')
         for scheme in schemes:
             for suffix in ("/sitemap.xml", "/sitemap_index.xml"):
                 sm = f"{scheme}://{host}{suffix}"
-                sm = await self._resolve_head(sm)
-                if sm:
-                    self._log("info", "Found sitemap at {url}", params={
-                              "url": sm}, tag="URL_SEED")
-                    async with aiofiles.open(path, "w") as fp:
+                resolved = await self._resolve_head(sm)
+                if resolved:
+                    sitemap_url = resolved
+                    # Fetch sitemap content to get lastmod
+                    try:
+                        r = await self.client.get(sitemap_url, timeout=15, follow_redirects=True)
+                        if 200 <= r.status_code < 300:
+                            sitemap_content = r.content
+                            sitemap_lastmod = _parse_sitemap_lastmod(sitemap_content)
+                    except Exception:
+                        pass
+                    break
+            if sitemap_url:
+                break
+
+        # Step 2: Check cache validity (skip if force=True)
+        if not force and cache_path.exists():
+            if _is_cache_valid(cache_path, cache_ttl_hours, validate_lastmod, sitemap_lastmod):
+                self._log("info", "Loading sitemap URLs from valid cache: {p}",
+                          params={"p": str(cache_path)}, tag="URL_SEED")
+                cached_urls = _read_cache(cache_path)
+                for url in cached_urls:
+                    if _match(url, pattern):
+                        yield url
+                return
+            else:
+                self._log("info", "Cache invalid/expired, refetching sitemap for {d}",
+                          params={"d": domain}, tag="URL_SEED")
+
+        # Step 3: Fetch fresh URLs
+        discovered_urls = []
+
+        if sitemap_url and sitemap_content:
+            self._log("info", "Found sitemap at {url}", params={"url": sitemap_url}, tag="URL_SEED")
+
+            # Parse sitemap (reuse content we already fetched)
+            async for u in self._iter_sitemap_content(sitemap_url, sitemap_content):
+                discovered_urls.append(u)
+                if _match(u, pattern):
+                    yield u
+        elif sitemap_url:
+            # We have a sitemap URL but no content (fetch failed earlier), try again
+            self._log("info", "Found sitemap at {url}", params={"url": sitemap_url}, tag="URL_SEED")
+            async for u in self._iter_sitemap(sitemap_url):
+                discovered_urls.append(u)
+                if _match(u, pattern):
+                    yield u
+        else:
+            # Fallback: robots.txt
+            robots = f"https://{host}/robots.txt"
+            try:
+                r = await self.client.get(robots, timeout=10, follow_redirects=True)
+                if 200 <= r.status_code < 300:
+                    sitemap_lines = [l.split(":", 1)[1].strip()
+                                     for l in r.text.splitlines()
+                                     if l.lower().startswith("sitemap:")]
+                    for sm in sitemap_lines:
                         async for u in self._iter_sitemap(sm):
-                            await fp.write(u + "\n")
+                            discovered_urls.append(u)
                             if _match(u, pattern):
                                 yield u
+                else:
+                    self._log("warning", "robots.txt unavailable for {d} HTTP{c}",
+                              params={"d": domain, "c": r.status_code}, tag="URL_SEED")
                     return
-
-        # 2️⃣ robots.txt fallback
-        robots = f"https://{domain.rstrip('/')}/robots.txt"
-        try:
-            r = await self.client.get(robots, timeout=10, follow_redirects=True)
-            if not 200 <= r.status_code < 300:
-                self._log("warning", "robots.txt unavailable for {d} HTTP{c}", params={
-                          "d": domain, "c": r.status_code}, tag="URL_SEED")
+            except Exception as e:
+                self._log("warning", "Failed to fetch robots.txt for {d}: {e}",
+                          params={"d": domain, "e": str(e)}, tag="URL_SEED")
                 return
-            sitemap_lines = [l.split(":", 1)[1].strip(
-            ) for l in r.text.splitlines() if l.lower().startswith("sitemap:")]
-        except Exception as e:
-            self._log("warning", "Failed to fetch robots.txt for {d}: {e}", params={
-                      "d": domain, "e": str(e)}, tag="URL_SEED")
-            return
 
-        if sitemap_lines:
-            async with aiofiles.open(path, "w") as fp:
-                for sm in sitemap_lines:
-                    async for u in self._iter_sitemap(sm):
-                        await fp.write(u + "\n")
-                        if _match(u, pattern):
-                            yield u
+        # Step 4: Write to cache (FALLBACK: if write fails, URLs still yielded above)
+        if discovered_urls:
+            _write_cache(cache_path, discovered_urls, sitemap_url or "", sitemap_lastmod)
+            self._log("info", "Cached {count} URLs for {d}",
+                      params={"count": len(discovered_urls), "d": domain}, tag="URL_SEED")
+
+    async def _iter_sitemap_content(self, url: str, content: bytes):
+        """Parse sitemap from already-fetched content."""
+        data = gzip.decompress(content) if url.endswith(".gz") else content
+        base_url = url
+
+        def _normalize_loc(raw: Optional[str]) -> Optional[str]:
+            if not raw:
+                return None
+            normalized = urljoin(base_url, raw.strip())
+            if not normalized:
+                return None
+            return normalized
+
+        # Detect if this is a sitemap index
+        is_sitemap_index = False
+        sub_sitemaps = []
+        regular_urls = []
+
+        if LXML:
+            try:
+                parser = etree.XMLParser(recover=True)
+                root = etree.fromstring(data, parser=parser)
+                sitemap_loc_nodes = root.xpath("//*[local-name()='sitemap']/*[local-name()='loc']")
+                url_loc_nodes = root.xpath("//*[local-name()='url']/*[local-name()='loc']")
+
+                if sitemap_loc_nodes:
+                    is_sitemap_index = True
+                    for sitemap_elem in sitemap_loc_nodes:
+                        loc = _normalize_loc(sitemap_elem.text)
+                        if loc:
+                            sub_sitemaps.append(loc)
+
+                if not is_sitemap_index:
+                    for loc_elem in url_loc_nodes:
+                        loc = _normalize_loc(loc_elem.text)
+                        if loc:
+                            regular_urls.append(loc)
+            except Exception as e:
+                self._log("error", "LXML parsing error for sitemap {url}: {error}",
+                          params={"url": url, "error": str(e)}, tag="URL_SEED")
+                return
+        else:
+            import xml.etree.ElementTree as ET
+            try:
+                root = ET.fromstring(data)
+                for elem in root.iter():
+                    if '}' in elem.tag:
+                        elem.tag = elem.tag.split('}')[1]
+
+                sitemaps = root.findall('.//sitemap')
+                url_entries = root.findall('.//url')
+
+                if sitemaps:
+                    is_sitemap_index = True
+                    for sitemap in sitemaps:
+                        loc_elem = sitemap.find('loc')
+                        loc = _normalize_loc(loc_elem.text if loc_elem is not None else None)
+                        if loc:
+                            sub_sitemaps.append(loc)
+
+                if not is_sitemap_index:
+                    for url_elem in url_entries:
+                        loc_elem = url_elem.find('loc')
+                        loc = _normalize_loc(loc_elem.text if loc_elem is not None else None)
+                        if loc:
+                            regular_urls.append(loc)
+            except Exception as e:
+                self._log("error", "ElementTree parsing error for sitemap {url}: {error}",
+                          params={"url": url, "error": str(e)}, tag="URL_SEED")
+                return
+
+        # Process based on type
+        if is_sitemap_index and sub_sitemaps:
+            self._log("info", "Processing sitemap index with {count} sub-sitemaps",
+                      params={"count": len(sub_sitemaps)}, tag="URL_SEED")
+
+            queue_size = min(50000, len(sub_sitemaps) * 1000)
+            result_queue = asyncio.Queue(maxsize=queue_size)
+            completed_count = 0
+            total_sitemaps = len(sub_sitemaps)
+
+            async def process_subsitemap(sitemap_url: str):
+                try:
+                    async for u in self._iter_sitemap(sitemap_url):
+                        await result_queue.put(u)
+                except Exception as e:
+                    self._log("error", "Error processing sub-sitemap {url}: {error}",
+                              params={"url": sitemap_url, "error": str(e)}, tag="URL_SEED")
+                finally:
+                    await result_queue.put(None)
+
+            tasks = [asyncio.create_task(process_subsitemap(sm)) for sm in sub_sitemaps]
+
+            while completed_count < total_sitemaps:
+                item = await result_queue.get()
+                if item is None:
+                    completed_count += 1
+                else:
+                    yield item
+
+            await asyncio.gather(*tasks, return_exceptions=True)
+        else:
+            for u in regular_urls:
+                yield u
 
     async def _iter_sitemap(self, url: str):
         try:
@@ -845,6 +1100,15 @@ class AsyncUrlSeeder:
             return
 
         data = gzip.decompress(r.content) if url.endswith(".gz") else r.content
+        base_url = str(r.url)
+
+        def _normalize_loc(raw: Optional[str]) -> Optional[str]:
+            if not raw:
+                return None
+            normalized = urljoin(base_url, raw.strip())
+            if not normalized:
+                return None
+            return normalized
 
         # Detect if this is a sitemap index by checking for <sitemapindex> or presence of <sitemap> elements
         is_sitemap_index = False
@@ -857,25 +1121,42 @@ class AsyncUrlSeeder:
                 # Use XML parser for sitemaps, not HTML parser
                 parser = etree.XMLParser(recover=True)
                 root = etree.fromstring(data, parser=parser)
+                # Namespace-agnostic lookups using local-name() so we honor custom or missing namespaces
+                sitemap_loc_nodes = root.xpath("//*[local-name()='sitemap']/*[local-name()='loc']")
+                url_loc_nodes = root.xpath("//*[local-name()='url']/*[local-name()='loc']")
 
-                # Define namespace for sitemap
-                ns = {'s': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
+                self._log(
+                    "debug",
+                    "Parsed sitemap {url}: {sitemap_count} sitemap entries, {url_count} url entries discovered",
+                    params={
+                        "url": url,
+                        "sitemap_count": len(sitemap_loc_nodes),
+                        "url_count": len(url_loc_nodes),
+                    },
+                    tag="URL_SEED",
+                )
 
                 # Check for sitemap index entries
-                sitemap_locs = root.xpath('//s:sitemap/s:loc', namespaces=ns)
-                if sitemap_locs:
+                if sitemap_loc_nodes:
                     is_sitemap_index = True
-                    for sitemap_elem in sitemap_locs:
-                        loc = sitemap_elem.text.strip() if sitemap_elem.text else ""
+                    for sitemap_elem in sitemap_loc_nodes:
+                        loc = _normalize_loc(sitemap_elem.text)
                         if loc:
                             sub_sitemaps.append(loc)
 
                 # If not a sitemap index, get regular URLs
                 if not is_sitemap_index:
-                    for loc_elem in root.xpath('//s:url/s:loc', namespaces=ns):
-                        loc = loc_elem.text.strip() if loc_elem.text else ""
+                    for loc_elem in url_loc_nodes:
+                        loc = _normalize_loc(loc_elem.text)
                         if loc:
                             regular_urls.append(loc)
+                    if not regular_urls:
+                        self._log(
+                            "warning",
+                            "No <loc> entries found inside <url> tags for sitemap {url}. The sitemap might be empty or use an unexpected structure.",
+                            params={"url": url},
+                            tag="URL_SEED",
+                        )
             except Exception as e:
                 self._log("error", "LXML parsing error for sitemap {url}: {error}",
                           params={"url": url, "error": str(e)}, tag="URL_SEED")
@@ -892,19 +1173,39 @@ class AsyncUrlSeeder:
 
                 # Check for sitemap index entries
                 sitemaps = root.findall('.//sitemap')
+                url_entries = root.findall('.//url')
+                self._log(
+                    "debug",
+                    "ElementTree parsed sitemap {url}: {sitemap_count} sitemap entries, {url_count} url entries discovered",
+                    params={
+                        "url": url,
+                        "sitemap_count": len(sitemaps),
+                        "url_count": len(url_entries),
+                    },
+                    tag="URL_SEED",
+                )
                 if sitemaps:
                     is_sitemap_index = True
                     for sitemap in sitemaps:
                         loc_elem = sitemap.find('loc')
-                        if loc_elem is not None and loc_elem.text:
-                            sub_sitemaps.append(loc_elem.text.strip())
+                        loc = _normalize_loc(loc_elem.text if loc_elem is not None else None)
+                        if loc:
+                            sub_sitemaps.append(loc)
 
                 # If not a sitemap index, get regular URLs
                 if not is_sitemap_index:
-                    for url_elem in root.findall('.//url'):
+                    for url_elem in url_entries:
                         loc_elem = url_elem.find('loc')
-                        if loc_elem is not None and loc_elem.text:
-                            regular_urls.append(loc_elem.text.strip())
+                        loc = _normalize_loc(loc_elem.text if loc_elem is not None else None)
+                        if loc:
+                            regular_urls.append(loc)
+                    if not regular_urls:
+                        self._log(
+                            "warning",
+                            "No <loc> entries found inside <url> tags for sitemap {url}. The sitemap might be empty or use an unexpected structure.",
+                            params={"url": url},
+                            tag="URL_SEED",
+                        )
             except Exception as e:
                 self._log("error", "ElementTree parsing error for sitemap {url}: {error}",
                           params={"url": url, "error": str(e)}, tag="URL_SEED")

crawl4ai/async_webcrawler.py

@@ -47,7 +47,9 @@ from .utils import (
     get_error_context,
     RobotsParser,
     preprocess_html_for_schema,
+    compute_head_fingerprint,
 )
+from .cache_validator import CacheValidator, CacheValidationResult
 
 
 class AsyncWebCrawler:
@@ -267,6 +269,51 @@ class AsyncWebCrawler:
                 if cache_context.should_read():
                     cached_result = await async_db_manager.aget_cached_url(url)
 
+                # Smart Cache: Validate cache freshness if enabled
+                if cached_result and config.check_cache_freshness:
+                    cache_metadata = await async_db_manager.aget_cache_metadata(url)
+                    if cache_metadata:
+                        async with CacheValidator(timeout=config.cache_validation_timeout) as validator:
+                            validation = await validator.validate(
+                                url=url,
+                                stored_etag=cache_metadata.get("etag"),
+                                stored_last_modified=cache_metadata.get("last_modified"),
+                                stored_head_fingerprint=cache_metadata.get("head_fingerprint"),
+                            )
+
+                        if validation.status == CacheValidationResult.FRESH:
+                            cached_result.cache_status = "hit_validated"
+                            self.logger.info(
+                                message="Cache validated: {reason}",
+                                tag="CACHE",
+                                params={"reason": validation.reason}
+                            )
+                            # Update metadata if we got new values
+                            if validation.new_etag or validation.new_last_modified:
+                                await async_db_manager.aupdate_cache_metadata(
+                                    url=url,
+                                    etag=validation.new_etag,
+                                    last_modified=validation.new_last_modified,
+                                    head_fingerprint=validation.new_head_fingerprint,
+                                )
+                        elif validation.status == CacheValidationResult.ERROR:
+                            cached_result.cache_status = "hit_fallback"
+                            self.logger.warning(
+                                message="Cache validation failed, using cached: {reason}",
+                                tag="CACHE",
+                                params={"reason": validation.reason}
+                            )
+                        else:
+                            # STALE or UNKNOWN - force recrawl
+                            self.logger.info(
+                                message="Cache stale: {reason}",
+                                tag="CACHE",
+                                params={"reason": validation.reason}
+                            )
+                            cached_result = None
+                elif cached_result:
+                    cached_result.cache_status = "hit"
+
                 if cached_result:
                     html = sanitize_input_encode(cached_result.html)
                     extracted_content = sanitize_input_encode(
@@ -296,15 +343,32 @@ class AsyncWebCrawler:
 
                 # Update proxy configuration from rotation strategy if available
                 if config and config.proxy_rotation_strategy:
-                    next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_next_proxy()
-                    if next_proxy:
-                        self.logger.info(
-                            message="Switch proxy: {proxy}",
-                            tag="PROXY",
-                            params={"proxy": next_proxy.server}
+                    # Handle sticky sessions - use same proxy for all requests with same session_id
+                    if config.proxy_session_id:
+                        next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_proxy_for_session(
+                            config.proxy_session_id,
+                            ttl=config.proxy_session_ttl
                         )
-                        config.proxy_config = next_proxy
-                        # config = config.clone(proxy_config=next_proxy)
+                        if next_proxy:
+                            self.logger.info(
+                                message="Using sticky proxy session: {session_id} -> {proxy}",
+                                tag="PROXY",
+                                params={
+                                    "session_id": config.proxy_session_id,
+                                    "proxy": next_proxy.server
+                                }
+                            )
+                            config.proxy_config = next_proxy
+                    else:
+                        # Existing behavior: rotate on each request
+                        next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_next_proxy()
+                        if next_proxy:
+                            self.logger.info(
+                                message="Switch proxy: {proxy}",
+                                tag="PROXY",
+                                params={"proxy": next_proxy.server}
+                            )
+                            config.proxy_config = next_proxy
 
                 # Fetch fresh content if needed
                 if not cached_result or not html:
@@ -383,6 +447,14 @@ class AsyncWebCrawler:
                     crawl_result.success = bool(html)
                     crawl_result.session_id = getattr(
                         config, "session_id", None)
+                    crawl_result.cache_status = "miss"
+
+                    # Compute head fingerprint for cache validation
+                    if html:
+                        head_end = html.lower().find('</head>')
+                        if head_end != -1:
+                            head_html = html[:head_end + 7]
+                            crawl_result.head_fingerprint = compute_head_fingerprint(head_html)
 
                     self.logger.url_status(
                         url=cache_context.display_url,
@@ -459,6 +531,27 @@ class AsyncWebCrawler:
         Returns:
             CrawlResult: Processed result containing extracted and formatted content
         """
+        # === PREFETCH MODE SHORT-CIRCUIT ===
+        if getattr(config, 'prefetch', False):
+            from .utils import quick_extract_links
+
+            # Use base_url from config (for raw: URLs), redirected_url, or original url
+            effective_url = getattr(config, 'base_url', None) or kwargs.get('redirected_url') or url
+            links = quick_extract_links(html, effective_url)
+
+            return CrawlResult(
+                url=url,
+                html=html,
+                success=True,
+                links=links,
+                status_code=kwargs.get('status_code'),
+                response_headers=kwargs.get('response_headers'),
+                redirected_url=kwargs.get('redirected_url'),
+                ssl_certificate=kwargs.get('ssl_certificate'),
+                # All other fields default to None
+            )
+        # === END PREFETCH SHORT-CIRCUIT ===
+
         cleaned_html = ""
         try:
             _url = url if not kwargs.get("is_raw_html", False) else "Raw HTML"
@@ -563,7 +656,8 @@ class AsyncWebCrawler:
         markdown_result: MarkdownGenerationResult = (
             markdown_generator.generate_markdown(
                 input_html=markdown_input_html,
-                base_url=params.get("redirected_url", url)
+                # Use explicit base_url if provided (for raw: HTML), otherwise redirected_url, then url
+                base_url=params.get("base_url") or params.get("redirected_url") or url
                 # html2text_options=kwargs.get('html2text', {})
             )
         )
@@ -617,7 +711,17 @@ class AsyncWebCrawler:
                 else config.chunking_strategy
             )
             sections = chunking.chunk(content)
-            extracted_content = config.extraction_strategy.run(url, sections)
+            # extracted_content = config.extraction_strategy.run(_url, sections)
+
+            # Use async version if available for better parallelism
+            if hasattr(config.extraction_strategy, 'arun'):
+                extracted_content = await config.extraction_strategy.arun(_url, sections)
+            else:
+                # Fallback to sync version run in thread pool to avoid blocking
+                extracted_content = await asyncio.to_thread(
+                    config.extraction_strategy.run, url, sections
+                )
+                
             extracted_content = json.dumps(
                 extracted_content, indent=4, default=str, ensure_ascii=False
             )
@@ -746,21 +850,45 @@ class AsyncWebCrawler:
         # Handle stream setting - use first config's stream setting if config is a list
         if isinstance(config, list):
             stream = config[0].stream if config else False
+            primary_config = config[0] if config else None
         else:
             stream = config.stream
+            primary_config = config
+
+        # Helper to release sticky session if auto_release is enabled
+        async def maybe_release_session():
+            if (primary_config and
+                primary_config.proxy_session_id and
+                primary_config.proxy_session_auto_release and
+                primary_config.proxy_rotation_strategy):
+                await primary_config.proxy_rotation_strategy.release_session(
+                    primary_config.proxy_session_id
+                )
+                self.logger.info(
+                    message="Auto-released proxy session: {session_id}",
+                    tag="PROXY",
+                    params={"session_id": primary_config.proxy_session_id}
+                )
 
         if stream:
-
             async def result_transformer():
-                async for task_result in dispatcher.run_urls_stream(
-                    crawler=self, urls=urls, config=config
-                ):
-                    yield transform_result(task_result)
+                try:
+                    async for task_result in dispatcher.run_urls_stream(
+                        crawler=self, urls=urls, config=config
+                    ):
+                        yield transform_result(task_result)
+                finally:
+                    # Auto-release session after streaming completes
+                    await maybe_release_session()
 
             return result_transformer()
         else:
-            _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config)
-            return [transform_result(res) for res in _results]
+            try:
+                _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config)
+                return [transform_result(res) for res in _results]
+            finally:
+                # Auto-release session after batch completes
+                await maybe_release_session()
 
     async def aseed_urls(
         self,

crawl4ai/browser_manager.py

@@ -369,6 +369,9 @@ class ManagedBrowser:
             ]
             if self.headless:
                 flags.append("--headless=new")
+            # Add viewport flag if specified in config
+            if self.browser_config.viewport_height and self.browser_config.viewport_width:
+                flags.append(f"--window-size={self.browser_config.viewport_width},{self.browser_config.viewport_height}")
             # merge common launch flags
             flags.extend(self.build_browser_flags(self.browser_config))
         elif self.browser_type == "firefox":
@@ -658,9 +661,44 @@ class BrowserManager:
         if self.config.cdp_url or self.config.use_managed_browser:
             self.config.use_managed_browser = True
             cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url
+
+            # Add CDP endpoint verification before connecting
+            if not await self._verify_cdp_ready(cdp_url):
+                raise Exception(f"CDP endpoint at {cdp_url} is not ready after startup")
+
             self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url)
             contexts = self.browser.contexts
-            if contexts:
+
+            # If browser_context_id is provided, we're using a pre-created context
+            if self.config.browser_context_id:
+                if self.logger:
+                    self.logger.debug(
+                        f"Using pre-existing browser context: {self.config.browser_context_id}",
+                        tag="BROWSER"
+                    )
+                # When connecting to a pre-created context, it should be in contexts
+                if contexts:
+                    self.default_context = contexts[0]
+                    if self.logger:
+                        self.logger.debug(
+                            f"Found {len(contexts)} existing context(s), using first one",
+                            tag="BROWSER"
+                        )
+                else:
+                    # Context was created but not yet visible - wait a bit
+                    await asyncio.sleep(0.2)
+                    contexts = self.browser.contexts
+                    if contexts:
+                        self.default_context = contexts[0]
+                    else:
+                        # Still no contexts - this shouldn't happen with pre-created context
+                        if self.logger:
+                            self.logger.warning(
+                                "Pre-created context not found, creating new one",
+                                tag="BROWSER"
+                            )
+                        self.default_context = await self.create_browser_context()
+            elif contexts:
                 self.default_context = contexts[0]
             else:
                 self.default_context = await self.create_browser_context()
@@ -678,6 +716,49 @@ class BrowserManager:
 
             self.default_context = self.browser
 
+    async def _verify_cdp_ready(self, cdp_url: str) -> bool:
+        """Verify CDP endpoint is ready with exponential backoff.
+
+        Supports multiple URL formats:
+        - HTTP URLs: http://localhost:9222
+        - HTTP URLs with query params: http://localhost:9222?browser_id=XXX
+        - WebSocket URLs: ws://localhost:9222/devtools/browser/XXX
+        """
+        import aiohttp
+        from urllib.parse import urlparse, urlunparse
+
+        # If WebSocket URL, Playwright handles connection directly - skip HTTP verification
+        if cdp_url.startswith(('ws://', 'wss://')):
+            self.logger.debug(f"WebSocket CDP URL provided, skipping HTTP verification", tag="BROWSER")
+            return True
+
+        # Parse HTTP URL and properly construct /json/version endpoint
+        parsed = urlparse(cdp_url)
+        # Build URL with /json/version path, preserving query params
+        verify_url = urlunparse((
+            parsed.scheme,
+            parsed.netloc,
+            '/json/version',  # Always use this path for verification
+            '',  # params
+            parsed.query,  # preserve query string
+            ''   # fragment
+        ))
+
+        self.logger.debug(f"Starting CDP verification for {verify_url}", tag="BROWSER")
+        for attempt in range(5):
+            try:
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(verify_url, timeout=aiohttp.ClientTimeout(total=2)) as response:
+                        if response.status == 200:
+                            self.logger.debug(f"CDP endpoint ready after {attempt + 1} attempts", tag="BROWSER")
+                            return True
+            except Exception as e:
+                self.logger.debug(f"CDP check attempt {attempt + 1} failed: {e}", tag="BROWSER")
+            delay = 0.5 * (1.4 ** attempt)
+            self.logger.debug(f"Waiting {delay:.2f}s before next CDP check...", tag="BROWSER")
+            await asyncio.sleep(delay)
+        self.logger.debug(f"CDP verification failed after 5 attempts", tag="BROWSER")
+        return False
 
     def _build_browser_args(self) -> dict:
         """Build browser launch arguments from config."""
@@ -814,18 +895,27 @@ class BrowserManager:
             combined_headers.update(self.config.headers)
             await context.set_extra_http_headers(combined_headers)
 
-        # Add default cookie
-        await context.add_cookies(
-            [
-                {
-                    "name": "cookiesEnabled",
-                    "value": "true",
-                    "url": crawlerRunConfig.url
-                    if crawlerRunConfig and crawlerRunConfig.url
-                    else "https://crawl4ai.com/",
-                }
-            ]
-        )
+        # Add default cookie (skip for raw:/file:// URLs which are not valid cookie URLs)
+        cookie_url = None
+        if crawlerRunConfig and crawlerRunConfig.url:
+            url = crawlerRunConfig.url
+            # Only set cookie for http/https URLs
+            if url.startswith(("http://", "https://")):
+                cookie_url = url
+            elif crawlerRunConfig.base_url and crawlerRunConfig.base_url.startswith(("http://", "https://")):
+                # Use base_url as fallback for raw:/file:// URLs
+                cookie_url = crawlerRunConfig.base_url
+
+        if cookie_url:
+            await context.add_cookies(
+                [
+                    {
+                        "name": "cookiesEnabled",
+                        "value": "true",
+                        "url": cookie_url,
+                    }
+                ]
+            )
 
         # Handle navigator overrides
         if crawlerRunConfig:
@@ -834,7 +924,12 @@ class BrowserManager:
                 or crawlerRunConfig.simulate_user
                 or crawlerRunConfig.magic
             ):
-                await context.add_init_script(load_js_script("navigator_overrider"))        
+                await context.add_init_script(load_js_script("navigator_overrider"))
+
+        # Apply custom init_scripts from BrowserConfig (for stealth evasions, etc.)
+        if self.config.init_scripts:
+            for script in self.config.init_scripts:
+                await context.add_init_script(script)
 
     async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None):
         """
@@ -1016,6 +1111,62 @@ class BrowserManager:
                         params={"error": str(e)}
                     )
 
+    async def _get_page_by_target_id(self, context: BrowserContext, target_id: str):
+        """
+        Get an existing page by its CDP target ID.
+
+        This is used when connecting to a pre-created browser context with an existing page.
+        Playwright may not immediately see targets created via raw CDP commands, so we
+        use CDP to get all targets and find the matching one.
+
+        Args:
+            context: The browser context to search in
+            target_id: The CDP target ID to find
+
+        Returns:
+            Page object if found, None otherwise
+        """
+        try:
+            # First check if Playwright already sees the page
+            for page in context.pages:
+                # Playwright's internal target ID might match
+                if hasattr(page, '_impl_obj') and hasattr(page._impl_obj, '_target_id'):
+                    if page._impl_obj._target_id == target_id:
+                        return page
+
+            # If not found, try using CDP to get targets
+            if hasattr(self.browser, '_impl_obj') and hasattr(self.browser._impl_obj, '_connection'):
+                cdp_session = await context.new_cdp_session(context.pages[0] if context.pages else None)
+                if cdp_session:
+                    try:
+                        result = await cdp_session.send("Target.getTargets")
+                        targets = result.get("targetInfos", [])
+                        for target in targets:
+                            if target.get("targetId") == target_id:
+                                # Found the target - if it's a page type, we can use it
+                                if target.get("type") == "page":
+                                    # The page exists, let Playwright discover it
+                                    await asyncio.sleep(0.1)
+                                    # Refresh pages list
+                                    if context.pages:
+                                        return context.pages[0]
+                    finally:
+                        await cdp_session.detach()
+
+            # Fallback: if there are any pages now, return the first one
+            if context.pages:
+                return context.pages[0]
+
+            return None
+        except Exception as e:
+            if self.logger:
+                self.logger.warning(
+                    message="Failed to get page by target ID: {error}",
+                    tag="BROWSER",
+                    params={"error": str(e)}
+                )
+            return None
+
     async def get_page(self, crawlerRunConfig: CrawlerRunConfig):
         """
         Get a page for the given session ID, creating a new one if needed.
@@ -1037,7 +1188,25 @@ class BrowserManager:
 
         # If using a managed browser, just grab the shared default_context
         if self.config.use_managed_browser:
-            if self.config.storage_state:
+            # If create_isolated_context is True, create isolated contexts for concurrent crawls
+            # Uses the same caching mechanism as non-CDP mode: cache context by config signature,
+            # but always create a new page. This prevents navigation conflicts while allowing
+            # context reuse for multiple URLs with the same config (e.g., batch/deep crawls).
+            if self.config.create_isolated_context:
+                config_signature = self._make_config_signature(crawlerRunConfig)
+
+                async with self._contexts_lock:
+                    if config_signature in self.contexts_by_config:
+                        context = self.contexts_by_config[config_signature]
+                    else:
+                        context = await self.create_browser_context(crawlerRunConfig)
+                        await self.setup_context(context, crawlerRunConfig)
+                        self.contexts_by_config[config_signature] = context
+
+                # Always create a new page for each crawl (isolation for navigation)
+                page = await context.new_page()
+                await self._apply_stealth_to_page(page)
+            elif self.config.storage_state:
                 context = await self.create_browser_context(crawlerRunConfig)
                 ctx = self.default_context        # default context, one window only
                 ctx = await clone_runtime_state(context, ctx, crawlerRunConfig, self.config)
@@ -1060,6 +1229,14 @@ class BrowserManager:
                             pages = context.pages
                             if pages:
                                 page = pages[0]
+                            elif self.config.browser_context_id and self.config.target_id:
+                                # Pre-existing context/target provided - use CDP to get the page
+                                # This handles the case where Playwright doesn't see the target yet
+                                page = await self._get_page_by_target_id(context, self.config.target_id)
+                                if not page:
+                                    # Fallback: create new page in existing context
+                                    page = await context.new_page()
+                                    await self._apply_stealth_to_page(page)
                             else:
                                 page = await context.new_page()
                                 await self._apply_stealth_to_page(page)
@@ -1114,8 +1291,44 @@ class BrowserManager:
     async def close(self):
         """Close all browser resources and clean up."""
         if self.config.cdp_url:
+            # When using external CDP, we don't own the browser process.
+            # If cdp_cleanup_on_close is True, properly disconnect from the browser
+            # and clean up Playwright resources. This frees the browser for other clients.
+            if self.config.cdp_cleanup_on_close:
+                # First close all sessions (pages)
+                session_ids = list(self.sessions.keys())
+                for session_id in session_ids:
+                    await self.kill_session(session_id)
+
+                # Close all contexts we created
+                for ctx in self.contexts_by_config.values():
+                    try:
+                        await ctx.close()
+                    except Exception:
+                        pass
+                self.contexts_by_config.clear()
+
+                # Disconnect from browser (doesn't terminate it, just releases connection)
+                if self.browser:
+                    try:
+                        await self.browser.close()
+                    except Exception as e:
+                        if self.logger:
+                            self.logger.debug(
+                                message="Error disconnecting from CDP browser: {error}",
+                                tag="BROWSER",
+                                params={"error": str(e)}
+                            )
+                    self.browser = None
+                    # Allow time for CDP connection to fully release before another client connects
+                    await asyncio.sleep(1.0)
+
+                # Stop Playwright instance to prevent memory leaks
+                if self.playwright:
+                    await self.playwright.stop()
+                    self.playwright = None
             return
-        
+
         if self.config.sleep_on_close:
             await asyncio.sleep(0.5)
 

crawl4ai/cache_validator.py

@@ -0,0 +1,270 @@
+"""
+Cache validation using HTTP conditional requests and head fingerprinting.
+
+Uses httpx for fast, lightweight HTTP requests (no browser needed).
+This module enables smart cache validation to avoid unnecessary full browser crawls
+when content hasn't changed.
+
+Validation Strategy:
+1. Send HEAD request with If-None-Match / If-Modified-Since headers
+2. If server returns 304 Not Modified → cache is FRESH
+3. If server returns 200 → fetch <head> and compare fingerprint
+4. If fingerprint matches → cache is FRESH (minor changes only)
+5. Otherwise → cache is STALE, need full recrawl
+"""
+
+import httpx
+from dataclasses import dataclass
+from typing import Optional, Tuple
+from enum import Enum
+
+from .utils import compute_head_fingerprint
+
+
+class CacheValidationResult(Enum):
+    """Result of cache validation check."""
+    FRESH = "fresh"       # Content unchanged, use cache
+    STALE = "stale"       # Content changed, need recrawl
+    UNKNOWN = "unknown"   # Couldn't determine, need recrawl
+    ERROR = "error"       # Request failed, use cache as fallback
+
+
+@dataclass
+class ValidationResult:
+    """Detailed result of a cache validation attempt."""
+    status: CacheValidationResult
+    new_etag: Optional[str] = None
+    new_last_modified: Optional[str] = None
+    new_head_fingerprint: Optional[str] = None
+    reason: str = ""
+
+
+class CacheValidator:
+    """
+    Validates cache freshness using lightweight HTTP requests.
+
+    This validator uses httpx to make fast HTTP requests without needing
+    a full browser. It supports two validation methods:
+
+    1. HTTP Conditional Requests (Layer 3):
+       - Uses If-None-Match with stored ETag
+       - Uses If-Modified-Since with stored Last-Modified
+       - Server returns 304 if content unchanged
+
+    2. Head Fingerprinting (Layer 4):
+       - Fetches only the <head> section (~5KB)
+       - Compares fingerprint of key meta tags
+       - Catches changes even without server support for conditional requests
+    """
+
+    def __init__(self, timeout: float = 10.0, user_agent: Optional[str] = None):
+        """
+        Initialize the cache validator.
+
+        Args:
+            timeout: Request timeout in seconds
+            user_agent: Custom User-Agent string (optional)
+        """
+        self.timeout = timeout
+        self.user_agent = user_agent or "Mozilla/5.0 (compatible; Crawl4AI/1.0)"
+        self._client: Optional[httpx.AsyncClient] = None
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        """Get or create the httpx client."""
+        if self._client is None:
+            self._client = httpx.AsyncClient(
+                http2=True,
+                timeout=self.timeout,
+                follow_redirects=True,
+                headers={"User-Agent": self.user_agent}
+            )
+        return self._client
+
+    async def validate(
+        self,
+        url: str,
+        stored_etag: Optional[str] = None,
+        stored_last_modified: Optional[str] = None,
+        stored_head_fingerprint: Optional[str] = None,
+    ) -> ValidationResult:
+        """
+        Validate if cached content is still fresh.
+
+        Args:
+            url: The URL to validate
+            stored_etag: Previously stored ETag header value
+            stored_last_modified: Previously stored Last-Modified header value
+            stored_head_fingerprint: Previously computed head fingerprint
+
+        Returns:
+            ValidationResult with status and any updated metadata
+        """
+        client = await self._get_client()
+
+        # Build conditional request headers
+        headers = {}
+        if stored_etag:
+            headers["If-None-Match"] = stored_etag
+        if stored_last_modified:
+            headers["If-Modified-Since"] = stored_last_modified
+
+        try:
+            # Step 1: Try HEAD request with conditional headers
+            if headers:
+                response = await client.head(url, headers=headers)
+
+                if response.status_code == 304:
+                    return ValidationResult(
+                        status=CacheValidationResult.FRESH,
+                        reason="Server returned 304 Not Modified"
+                    )
+
+                # Got 200, extract new headers for potential update
+                new_etag = response.headers.get("etag")
+                new_last_modified = response.headers.get("last-modified")
+
+                # If we have fingerprint, compare it
+                if stored_head_fingerprint:
+                    head_html, _, _ = await self._fetch_head(url)
+                    if head_html:
+                        new_fingerprint = compute_head_fingerprint(head_html)
+                        if new_fingerprint and new_fingerprint == stored_head_fingerprint:
+                            return ValidationResult(
+                                status=CacheValidationResult.FRESH,
+                                new_etag=new_etag,
+                                new_last_modified=new_last_modified,
+                                new_head_fingerprint=new_fingerprint,
+                                reason="Head fingerprint matches"
+                            )
+                        elif new_fingerprint:
+                            return ValidationResult(
+                                status=CacheValidationResult.STALE,
+                                new_etag=new_etag,
+                                new_last_modified=new_last_modified,
+                                new_head_fingerprint=new_fingerprint,
+                                reason="Head fingerprint changed"
+                            )
+
+                # Headers changed and no fingerprint match
+                return ValidationResult(
+                    status=CacheValidationResult.STALE,
+                    new_etag=new_etag,
+                    new_last_modified=new_last_modified,
+                    reason="Server returned 200, content may have changed"
+                )
+
+            # Step 2: No conditional headers available, try fingerprint only
+            if stored_head_fingerprint:
+                head_html, new_etag, new_last_modified = await self._fetch_head(url)
+
+                if head_html:
+                    new_fingerprint = compute_head_fingerprint(head_html)
+
+                    if new_fingerprint and new_fingerprint == stored_head_fingerprint:
+                        return ValidationResult(
+                            status=CacheValidationResult.FRESH,
+                            new_etag=new_etag,
+                            new_last_modified=new_last_modified,
+                            new_head_fingerprint=new_fingerprint,
+                            reason="Head fingerprint matches"
+                        )
+                    elif new_fingerprint:
+                        return ValidationResult(
+                            status=CacheValidationResult.STALE,
+                            new_etag=new_etag,
+                            new_last_modified=new_last_modified,
+                            new_head_fingerprint=new_fingerprint,
+                            reason="Head fingerprint changed"
+                        )
+
+            # Step 3: No validation data available
+            return ValidationResult(
+                status=CacheValidationResult.UNKNOWN,
+                reason="No validation data available (no etag, last-modified, or fingerprint)"
+            )
+
+        except httpx.TimeoutException:
+            return ValidationResult(
+                status=CacheValidationResult.ERROR,
+                reason="Validation request timed out"
+            )
+        except httpx.RequestError as e:
+            return ValidationResult(
+                status=CacheValidationResult.ERROR,
+                reason=f"Validation request failed: {type(e).__name__}"
+            )
+        except Exception as e:
+            # On unexpected error, prefer using cache over failing
+            return ValidationResult(
+                status=CacheValidationResult.ERROR,
+                reason=f"Validation error: {str(e)}"
+            )
+
+    async def _fetch_head(self, url: str) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+        """
+        Fetch only the <head> section of a page.
+
+        Uses streaming to stop reading after </head> is found,
+        minimizing bandwidth usage.
+
+        Args:
+            url: The URL to fetch
+
+        Returns:
+            Tuple of (head_html, etag, last_modified)
+        """
+        client = await self._get_client()
+
+        try:
+            async with client.stream(
+                "GET",
+                url,
+                headers={"Accept-Encoding": "identity"}  # Disable compression for easier parsing
+            ) as response:
+                etag = response.headers.get("etag")
+                last_modified = response.headers.get("last-modified")
+
+                if response.status_code != 200:
+                    return None, etag, last_modified
+
+                # Read until </head> or max 64KB
+                chunks = []
+                total_bytes = 0
+                max_bytes = 65536
+
+                async for chunk in response.aiter_bytes(4096):
+                    chunks.append(chunk)
+                    total_bytes += len(chunk)
+
+                    content = b''.join(chunks)
+                    # Check for </head> (case insensitive)
+                    if b'</head>' in content.lower() or b'</HEAD>' in content:
+                        break
+                    if total_bytes >= max_bytes:
+                        break
+
+                html = content.decode('utf-8', errors='replace')
+
+                # Extract just the head section
+                head_end = html.lower().find('</head>')
+                if head_end != -1:
+                    html = html[:head_end + 7]
+
+                return html, etag, last_modified
+
+        except Exception:
+            return None, None, None
+
+    async def close(self):
+        """Close the HTTP client and release resources."""
+        if self._client:
+            await self._client.aclose()
+            self._client = None
+
+    async def __aenter__(self):
+        """Async context manager entry."""
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self.close()

crawl4ai/content_filter_strategy.py

@@ -980,6 +980,9 @@ class LLMContentFilter(RelevantContentFilter):
                         prompt,
                         api_token,
                         base_url=base_url,
+                        base_delay=self.llm_config.backoff_base_delay,
+                        max_attempts=self.llm_config.backoff_max_attempts,
+                        exponential_factor=self.llm_config.backoff_exponential_factor,
                         extra_args=extra_args,
                     )
 

crawl4ai/content_scraping_strategy.py

@@ -542,6 +542,19 @@ class LXMLWebScrapingStrategy(ContentScrapingStrategy):
             if el.tag in bypass_tags:
                 continue
 
+            # Skip elements inside <pre> or <code> tags where whitespace is significant
+            # This preserves whitespace-only spans (e.g., <span class="w"> </span>) in code blocks
+            is_in_code_block = False
+            ancestor = el.getparent()
+            while ancestor is not None:
+                if ancestor.tag in ("pre", "code"):
+                    is_in_code_block = True
+                    break
+                ancestor = ancestor.getparent()
+
+            if is_in_code_block:
+                continue
+
             text_content = (el.text_content() or "").strip()
             if (
                 len(text_content.split()) < word_count_threshold

crawl4ai/deep_crawling/bff_strategy.py

@@ -2,7 +2,7 @@
 import asyncio
 import logging
 from datetime import datetime
-from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple, Any, Callable, Awaitable
 from urllib.parse import urlparse
 
 from ..models import TraversalStats
@@ -41,6 +41,9 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
         include_external: bool = False,
         max_pages: int = infinity,
         logger: Optional[logging.Logger] = None,
+        # Optional resume/callback parameters for crash recovery
+        resume_state: Optional[Dict[str, Any]] = None,
+        on_state_change: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
     ):
         self.max_depth = max_depth
         self.filter_chain = filter_chain
@@ -57,6 +60,12 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
         self.stats = TraversalStats(start_time=datetime.now())
         self._cancel_event = asyncio.Event()
         self._pages_crawled = 0
+        # Store for use in arun methods
+        self._resume_state = resume_state
+        self._on_state_change = on_state_change
+        self._last_state: Optional[Dict[str, Any]] = None
+        # Shadow list for queue items (only used when on_state_change is set)
+        self._queue_shadow: Optional[List[Tuple[float, int, str, Optional[str]]]] = None
 
     async def can_process_url(self, url: str, depth: int) -> bool:
         """
@@ -135,16 +144,36 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
     ) -> AsyncGenerator[CrawlResult, None]:
         """
         Core best-first crawl method using a priority queue.
-        
+
         The queue items are tuples of (score, depth, url, parent_url). Lower scores
         are treated as higher priority. URLs are processed in batches for efficiency.
         """
         queue: asyncio.PriorityQueue = asyncio.PriorityQueue()
-        # Push the initial URL with score 0 and depth 0.
-        initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0
-        await queue.put((-initial_score, 0, start_url, None))
-        visited: Set[str] = set()
-        depths: Dict[str, int] = {start_url: 0}
+
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+            # Restore queue from saved items
+            queue_items = self._resume_state.get("queue_items", [])
+            for item in queue_items:
+                await queue.put((item["score"], item["depth"], item["url"], item["parent_url"]))
+            # Initialize shadow list if callback is set
+            if self._on_state_change:
+                self._queue_shadow = [
+                    (item["score"], item["depth"], item["url"], item["parent_url"])
+                    for item in queue_items
+                ]
+        else:
+            # Original initialization
+            initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0
+            await queue.put((-initial_score, 0, start_url, None))
+            visited: Set[str] = set()
+            depths: Dict[str, int] = {start_url: 0}
+            # Initialize shadow list if callback is set
+            if self._on_state_change:
+                self._queue_shadow = [(-initial_score, 0, start_url, None)]
 
         while not queue.empty() and not self._cancel_event.is_set():
             # Stop if we've reached the max pages limit
@@ -166,6 +195,12 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
                 if queue.empty():
                     break
                 item = await queue.get()
+                # Remove from shadow list if tracking
+                if self._on_state_change and self._queue_shadow is not None:
+                    try:
+                        self._queue_shadow.remove(item)
+                    except ValueError:
+                        pass  # Item may have been removed already
                 score, depth, url, parent_url = item
                 if url in visited:
                     continue
@@ -210,7 +245,26 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
                     for new_url, new_parent in new_links:
                         new_depth = depths.get(new_url, depth + 1)
                         new_score = self.url_scorer.score(new_url) if self.url_scorer else 0
-                        await queue.put((-new_score, new_depth, new_url, new_parent))
+                        queue_item = (-new_score, new_depth, new_url, new_parent)
+                        await queue.put(queue_item)
+                        # Add to shadow list if tracking
+                        if self._on_state_change and self._queue_shadow is not None:
+                            self._queue_shadow.append(queue_item)
+
+                    # Capture state after EACH URL processed (if callback set)
+                    if self._on_state_change and self._queue_shadow is not None:
+                        state = {
+                            "strategy_type": "best_first",
+                            "visited": list(visited),
+                            "queue_items": [
+                                {"score": s, "depth": d, "url": u, "parent_url": p}
+                                for s, d, u, p in self._queue_shadow
+                            ],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
 
         # End of crawl.
 
@@ -269,3 +323,15 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
         """
         self._cancel_event.set()
         self.stats.end_time = datetime.now()
+
+    def export_state(self) -> Optional[Dict[str, Any]]:
+        """
+        Export current crawl state for external persistence.
+
+        Note: This returns the last captured state. For real-time state,
+        use the on_state_change callback.
+
+        Returns:
+            Dict with strategy state, or None if no state captured yet.
+        """
+        return self._last_state

crawl4ai/deep_crawling/bfs_strategy.py

@@ -2,7 +2,7 @@
 import asyncio
 import logging
 from datetime import datetime
-from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
+from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple, Any, Callable, Awaitable
 from urllib.parse import urlparse
 
 from ..models import TraversalStats
@@ -26,11 +26,14 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         self,
         max_depth: int,
         filter_chain: FilterChain = FilterChain(),
-        url_scorer: Optional[URLScorer] = None,        
+        url_scorer: Optional[URLScorer] = None,
         include_external: bool = False,
         score_threshold: float = -infinity,
         max_pages: int = infinity,
         logger: Optional[logging.Logger] = None,
+        # Optional resume/callback parameters for crash recovery
+        resume_state: Optional[Dict[str, Any]] = None,
+        on_state_change: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
     ):
         self.max_depth = max_depth
         self.filter_chain = filter_chain
@@ -48,6 +51,10 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         self.stats = TraversalStats(start_time=datetime.now())
         self._cancel_event = asyncio.Event()
         self._pages_crawled = 0
+        # Store for use in arun methods
+        self._resume_state = resume_state
+        self._on_state_change = on_state_change
+        self._last_state: Optional[Dict[str, Any]] = None
 
     async def can_process_url(self, url: str, depth: int) -> bool:
         """
@@ -155,10 +162,21 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         Batch (non-streaming) mode:
         Processes one BFS level at a time, then yields all the results.
         """
-        visited: Set[str] = set()
-        # current_level holds tuples: (url, parent_url)
-        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
-        depths: Dict[str, int] = {start_url: 0}
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            current_level = [
+                (item["url"], item["parent_url"])
+                for item in self._resume_state.get("pending", [])
+            ]
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+        else:
+            # Original initialization
+            visited: Set[str] = set()
+            # current_level holds tuples: (url, parent_url)
+            current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+            depths: Dict[str, int] = {start_url: 0}
 
         results: List[CrawlResult] = []
 
@@ -174,11 +192,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
             # Clone the config to disable deep crawling recursion and enforce batch mode.
             batch_config = config.clone(deep_crawl_strategy=None, stream=False)
             batch_results = await crawler.arun_many(urls=urls, config=batch_config)
-            
-            # Update pages crawled counter - count only successful crawls
-            successful_results = [r for r in batch_results if r.success]
-            self._pages_crawled += len(successful_results)
-            
+
             for result in batch_results:
                 url = result.url
                 depth = depths.get(url, 0)
@@ -187,12 +201,27 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
                 parent_url = next((parent for (u, parent) in current_level if u == url), None)
                 result.metadata["parent_url"] = parent_url
                 results.append(result)
-                
+
                 # Only discover links from successful crawls
                 if result.success:
+                    # Increment pages crawled per URL for accurate state tracking
+                    self._pages_crawled += 1
+
                     # Link discovery will handle the max pages limit internally
                     await self.link_discovery(result, url, depth, visited, next_level, depths)
 
+                    # Capture state after EACH URL processed (if callback set)
+                    if self._on_state_change:
+                        state = {
+                            "strategy_type": "bfs",
+                            "visited": list(visited),
+                            "pending": [{"url": u, "parent_url": p} for u, p in next_level],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
+
             current_level = next_level
 
         return results
@@ -207,9 +236,20 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         Streaming mode:
         Processes one BFS level at a time and yields results immediately as they arrive.
         """
-        visited: Set[str] = set()
-        current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
-        depths: Dict[str, int] = {start_url: 0}
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            current_level = [
+                (item["url"], item["parent_url"])
+                for item in self._resume_state.get("pending", [])
+            ]
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+        else:
+            # Original initialization
+            visited: Set[str] = set()
+            current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
+            depths: Dict[str, int] = {start_url: 0}
 
         while current_level and not self._cancel_event.is_set():
             next_level: List[Tuple[str, Optional[str]]] = []
@@ -244,7 +284,19 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
                 if result.success:
                     # Link discovery will handle the max pages limit internally
                     await self.link_discovery(result, url, depth, visited, next_level, depths)
-            
+
+                    # Capture state after EACH URL processed (if callback set)
+                    if self._on_state_change:
+                        state = {
+                            "strategy_type": "bfs",
+                            "visited": list(visited),
+                            "pending": [{"url": u, "parent_url": p} for u, p in next_level],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
+
             # If we didn't get results back (e.g. due to errors), avoid getting stuck in an infinite loop
             # by considering these URLs as visited but not counting them toward the max_pages limit
             if results_count == 0 and urls:
@@ -258,3 +310,15 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
         """
         self._cancel_event.set()
         self.stats.end_time = datetime.now()
+
+    def export_state(self) -> Optional[Dict[str, Any]]:
+        """
+        Export current crawl state for external persistence.
+
+        Note: This returns the last captured state. For real-time state,
+        use the on_state_change callback.
+
+        Returns:
+            Dict with strategy state, or None if no state captured yet.
+        """
+        return self._last_state

crawl4ai/deep_crawling/dfs_strategy.py

@@ -4,14 +4,26 @@ from typing import AsyncGenerator, Optional, Set, Dict, List, Tuple
 from ..models import CrawlResult
 from .bfs_strategy import BFSDeepCrawlStrategy  # noqa
 from ..types import AsyncWebCrawler, CrawlerRunConfig
+from ..utils import normalize_url_for_deep_crawl
 
 class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
     """
-    Depth-First Search (DFS) deep crawling strategy.
+    Depth-first deep crawling with familiar BFS rules.
 
-    Inherits URL validation and link discovery from BFSDeepCrawlStrategy.
-    Overrides _arun_batch and _arun_stream to use a stack (LIFO) for DFS traversal.
+    We reuse the same filters, scoring, and page limits from :class:`BFSDeepCrawlStrategy`,
+    but walk the graph with a stack so we fully explore one branch before hopping to the
+    next. DFS also keeps its own ``_dfs_seen`` set so we can drop duplicate links at
+    discovery time without accidentally marking them as “already crawled”.
     """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._dfs_seen: Set[str] = set()
+
+    def _reset_seen(self, start_url: str) -> None:
+        """Start each crawl with a clean dedupe set seeded with the root URL."""
+        self._dfs_seen = {start_url}
+
     async def _arun_batch(
         self,
         start_url: str,
@@ -19,14 +31,32 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
         config: CrawlerRunConfig,
     ) -> List[CrawlResult]:
         """
-        Batch (non-streaming) DFS mode.
-        Uses a stack to traverse URLs in DFS order, aggregating CrawlResults into a list.
+        Crawl level-by-level but emit results at the end.
+
+        We keep a stack of ``(url, parent, depth)`` tuples, pop one at a time, and
+        hand it to ``crawler.arun_many`` with deep crawling disabled so we remain
+        in control of traversal. Every successful page bumps ``_pages_crawled`` and
+        seeds new stack items discovered via :meth:`link_discovery`.
         """
-        visited: Set[str] = set()
-        # Stack items: (url, parent_url, depth)
-        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
-        depths: Dict[str, int] = {start_url: 0}
-        results: List[CrawlResult] = []
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            stack = [
+                (item["url"], item["parent_url"], item["depth"])
+                for item in self._resume_state.get("stack", [])
+            ]
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+            self._dfs_seen = set(self._resume_state.get("dfs_seen", []))
+            results: List[CrawlResult] = []
+        else:
+            # Original initialization
+            visited: Set[str] = set()
+            # Stack items: (url, parent_url, depth)
+            stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+            depths: Dict[str, int] = {start_url: 0}
+            results: List[CrawlResult] = []
+            self._reset_seen(start_url)
 
         while stack and not self._cancel_event.is_set():
             url, parent, depth = stack.pop()
@@ -62,6 +92,22 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
                     for new_url, new_parent in reversed(new_links):
                         new_depth = depths.get(new_url, depth + 1)
                         stack.append((new_url, new_parent, new_depth))
+
+                    # Capture state after each URL processed (if callback set)
+                    if self._on_state_change:
+                        state = {
+                            "strategy_type": "dfs",
+                            "visited": list(visited),
+                            "stack": [
+                                {"url": u, "parent_url": p, "depth": d}
+                                for u, p, d in stack
+                            ],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                            "dfs_seen": list(self._dfs_seen),
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
         return results
 
     async def _arun_stream(
@@ -71,12 +117,28 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
         config: CrawlerRunConfig,
     ) -> AsyncGenerator[CrawlResult, None]:
         """
-        Streaming DFS mode.
-        Uses a stack to traverse URLs in DFS order and yields CrawlResults as they become available.
+        Same traversal as :meth:`_arun_batch`, but yield pages immediately.
+
+        Each popped URL is crawled, its metadata annotated, then the result gets
+        yielded before we even look at the next stack entry. Successful crawls
+        still feed :meth:`link_discovery`, keeping DFS order intact.
         """
-        visited: Set[str] = set()
-        stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
-        depths: Dict[str, int] = {start_url: 0}
+        # Conditional state initialization for resume support
+        if self._resume_state:
+            visited = set(self._resume_state.get("visited", []))
+            stack = [
+                (item["url"], item["parent_url"], item["depth"])
+                for item in self._resume_state.get("stack", [])
+            ]
+            depths = dict(self._resume_state.get("depths", {}))
+            self._pages_crawled = self._resume_state.get("pages_crawled", 0)
+            self._dfs_seen = set(self._resume_state.get("dfs_seen", []))
+        else:
+            # Original initialization
+            visited: Set[str] = set()
+            stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
+            depths: Dict[str, int] = {start_url: 0}
+            self._reset_seen(start_url)
 
         while stack and not self._cancel_event.is_set():
             url, parent, depth = stack.pop()
@@ -108,3 +170,108 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
                     for new_url, new_parent in reversed(new_links):
                         new_depth = depths.get(new_url, depth + 1)
                         stack.append((new_url, new_parent, new_depth))
+
+                    # Capture state after each URL processed (if callback set)
+                    if self._on_state_change:
+                        state = {
+                            "strategy_type": "dfs",
+                            "visited": list(visited),
+                            "stack": [
+                                {"url": u, "parent_url": p, "depth": d}
+                                for u, p, d in stack
+                            ],
+                            "depths": depths,
+                            "pages_crawled": self._pages_crawled,
+                            "dfs_seen": list(self._dfs_seen),
+                        }
+                        self._last_state = state
+                        await self._on_state_change(state)
+
+    async def link_discovery(
+        self,
+        result: CrawlResult,
+        source_url: str,
+        current_depth: int,
+        _visited: Set[str],
+        next_level: List[Tuple[str, Optional[str]]],
+        depths: Dict[str, int],
+    ) -> None:
+        """
+        Find the next URLs we should push onto the DFS stack.
+
+        Parameters
+        ----------
+        result : CrawlResult
+            Output of the page we just crawled; its ``links`` block is our raw material.
+        source_url : str
+            URL of the parent page; stored so callers can track ancestry.
+        current_depth : int
+            Depth of the parent; children naturally sit at ``current_depth + 1``.
+        _visited : Set[str]
+            Present to match the BFS signature, but we rely on ``_dfs_seen`` instead.
+        next_level : list of tuples
+            The stack buffer supplied by the caller; we append new ``(url, parent)`` items here.
+        depths : dict
+            Shared depth map so future metadata tagging knows how deep each URL lives.
+
+        Notes
+        -----
+        - ``_dfs_seen`` keeps us from pushing duplicates without touching the traversal guard.
+        - Validation, scoring, and capacity trimming mirror the BFS version so behaviour stays consistent.
+        """
+        next_depth = current_depth + 1
+        if next_depth > self.max_depth:
+            return
+
+        remaining_capacity = self.max_pages - self._pages_crawled
+        if remaining_capacity <= 0:
+            self.logger.info(
+                f"Max pages limit ({self.max_pages}) reached, stopping link discovery"
+            )
+            return
+
+        links = result.links.get("internal", [])
+        if self.include_external:
+            links += result.links.get("external", [])
+
+        seen = self._dfs_seen
+        valid_links: List[Tuple[str, float]] = []
+
+        for link in links:
+            raw_url = link.get("href")
+            if not raw_url:
+                continue
+
+            normalized_url = normalize_url_for_deep_crawl(raw_url, source_url)
+            if not normalized_url or normalized_url in seen:
+                continue
+
+            if not await self.can_process_url(raw_url, next_depth):
+                self.stats.urls_skipped += 1
+                continue
+
+            score = self.url_scorer.score(normalized_url) if self.url_scorer else 0
+            if score < self.score_threshold:
+                self.logger.debug(
+                    f"URL {normalized_url} skipped: score {score} below threshold {self.score_threshold}"
+                )
+                self.stats.urls_skipped += 1
+                continue
+
+            seen.add(normalized_url)
+            valid_links.append((normalized_url, score))
+
+        if len(valid_links) > remaining_capacity:
+            if self.url_scorer:
+                valid_links.sort(key=lambda x: x[1], reverse=True)
+            valid_links = valid_links[:remaining_capacity]
+            self.logger.info(
+                f"Limiting to {remaining_capacity} URLs due to max_pages limit"
+            )
+
+        for url, score in valid_links:
+            if score:
+                result.metadata = result.metadata or {}
+                result.metadata["score"] = score
+            next_level.append((url, source_url))
+            depths[url] = next_depth

crawl4ai/deep_crawling/filters.py

@@ -509,18 +509,22 @@ class DomainFilter(URLFilter):
 class ContentRelevanceFilter(URLFilter):
     """BM25-based relevance filter using head section content"""
 
-    __slots__ = ("query_terms", "threshold", "k1", "b", "avgdl")
+    __slots__ = ("query_terms", "threshold", "k1", "b", "avgdl", "query")
 
     def __init__(
         self,
-        query: str,
+        query: Union[str, List[str]],
         threshold: float,
         k1: float = 1.2,
         b: float = 0.75,
         avgdl: int = 1000,
     ):
         super().__init__(name="BM25RelevanceFilter")
-        self.query_terms = self._tokenize(query)
+        if isinstance(query, list):
+            self.query = " ".join(query)
+        else:
+            self.query = query
+        self.query_terms = self._tokenize(self.query)
         self.threshold = threshold
         self.k1 = k1  # TF saturation parameter
         self.b = b  # Length normalization parameter

crawl4ai/docker_client.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Union, AsyncGenerator, Dict, Any
+from typing import List, Optional, Union, AsyncGenerator, Dict, Any, Callable
 import httpx
 import json
 from urllib.parse import urljoin
@@ -7,6 +7,7 @@ import asyncio
 from .async_configs import BrowserConfig, CrawlerRunConfig
 from .models import CrawlResult
 from .async_logger import AsyncLogger, LogLevel
+from .utils import hooks_to_string
 
 
 class Crawl4aiClientError(Exception):
@@ -70,17 +71,41 @@ class Crawl4aiDockerClient:
             self.logger.error(f"Server unreachable: {str(e)}", tag="ERROR")
             raise ConnectionError(f"Cannot connect to server: {str(e)}")
 
-    def _prepare_request(self, urls: List[str], browser_config: Optional[BrowserConfig] = None, 
-                       crawler_config: Optional[CrawlerRunConfig] = None) -> Dict[str, Any]:
+    def _prepare_request(
+        self,
+        urls: List[str],
+        browser_config: Optional[BrowserConfig] = None,
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None,
+        hooks_timeout: int = 30
+    ) -> Dict[str, Any]:
         """Prepare request data from configs."""
         if self._token:
             self._http_client.headers["Authorization"] = f"Bearer {self._token}"
-        return {
+
+        request_data = {
             "urls": urls,
             "browser_config": browser_config.dump() if browser_config else {},
             "crawler_config": crawler_config.dump() if crawler_config else {}
         }
 
+        # Handle hooks if provided
+        if hooks:
+            # Check if hooks are already strings or need conversion
+            if any(callable(v) for v in hooks.values()):
+                # Convert function objects to strings
+                hooks_code = hooks_to_string(hooks)
+            else:
+                # Already in string format
+                hooks_code = hooks
+
+            request_data["hooks"] = {
+                "code": hooks_code,
+                "timeout": hooks_timeout
+            }
+
+        return request_data
+
     async def _request(self, method: str, endpoint: str, **kwargs) -> httpx.Response:
         """Make an HTTP request with error handling."""
         url = urljoin(self.base_url, endpoint)
@@ -102,16 +127,42 @@ class Crawl4aiDockerClient:
         self,
         urls: List[str],
         browser_config: Optional[BrowserConfig] = None,
-        crawler_config: Optional[CrawlerRunConfig] = None
+        crawler_config: Optional[CrawlerRunConfig] = None,
+        hooks: Optional[Union[Dict[str, Callable], Dict[str, str]]] = None,
+        hooks_timeout: int = 30
     ) -> Union[CrawlResult, List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
-        """Execute a crawl operation."""
+        """
+        Execute a crawl operation.
+
+        Args:
+            urls: List of URLs to crawl
+            browser_config: Browser configuration
+            crawler_config: Crawler configuration
+            hooks: Optional hooks - can be either:
+                   - Dict[str, Callable]: Function objects that will be converted to strings
+                   - Dict[str, str]: Already stringified hook code
+            hooks_timeout: Timeout in seconds for each hook execution (1-120)
+
+        Returns:
+            Single CrawlResult, list of results, or async generator for streaming
+
+        Example with function hooks:
+            >>> async def my_hook(page, context, **kwargs):
+            ...     await page.set_viewport_size({"width": 1920, "height": 1080})
+            ...     return page
+            >>>
+            >>> result = await client.crawl(
+            ...     ["https://example.com"],
+            ...     hooks={"on_page_context_created": my_hook}
+            ... )
+        """
         await self._check_server()
-        
-        data = self._prepare_request(urls, browser_config, crawler_config)
+
+        data = self._prepare_request(urls, browser_config, crawler_config, hooks, hooks_timeout)
         is_streaming = crawler_config and crawler_config.stream
-        
+
         self.logger.info(f"Crawling {len(urls)} URLs {'(streaming)' if is_streaming else ''}", tag="CRAWL")
-        
+
         if is_streaming:
             async def stream_results() -> AsyncGenerator[CrawlResult, None]:
                 async with self._http_client.stream("POST", f"{self.base_url}/crawl/stream", json=data) as response:
@@ -128,12 +179,12 @@ class Crawl4aiDockerClient:
                             else:
                                 yield CrawlResult(**result)
             return stream_results()
-        
-        response = await self._request("POST", "/crawl", json=data)
+
+        response = await self._request("POST", "/crawl", json=data, timeout=hooks_timeout)
         result_data = response.json()
         if not result_data.get("success", False):
             raise RequestError(f"Crawl failed: {result_data.get('msg', 'Unknown error')}")
-        
+
         results = [CrawlResult(**r) for r in result_data.get("results", [])]
         self.logger.success(f"Crawl completed with {len(results)} results", tag="CRAWL")
         return results[0] if len(results) == 1 else results

crawl4ai/extraction_strategy.py

@@ -94,6 +94,20 @@ class ExtractionStrategy(ABC):
                 extracted_content.extend(future.result())
         return extracted_content
 
+    async def arun(self, url: str, sections: List[str], *q, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Async version: Process sections of text in parallel using asyncio.
+
+        Default implementation runs the sync version in a thread pool.
+        Subclasses can override this for true async processing.
+
+        :param url: The URL of the webpage.
+        :param sections: List of sections (strings) to process.
+        :return: A list of processed JSON blocks.
+        """
+        import asyncio
+        return await asyncio.to_thread(self.run, url, sections, *q, **kwargs)
+
 
 class NoExtractionStrategy(ExtractionStrategy):
     """
@@ -635,6 +649,9 @@ class LLMExtractionStrategy(ExtractionStrategy):
                 base_url=self.llm_config.base_url,
                 json_response=self.force_json_response,
                 extra_args=self.extra_args,
+                base_delay=self.llm_config.backoff_base_delay,
+                max_attempts=self.llm_config.backoff_max_attempts,
+                exponential_factor=self.llm_config.backoff_exponential_factor
             )  # , json_response=self.extract_type == "schema")
             # Track usage
             usage = TokenUsage(
@@ -780,6 +797,180 @@ class LLMExtractionStrategy(ExtractionStrategy):
 
         return extracted_content
 
+    async def aextract(self, url: str, ix: int, html: str) -> List[Dict[str, Any]]:
+        """
+        Async version: Extract meaningful blocks or chunks from the given HTML using an LLM.
+
+        How it works:
+        1. Construct a prompt with variables.
+        2. Make an async request to the LLM using the prompt.
+        3. Parse the response and extract blocks or chunks.
+
+        Args:
+            url: The URL of the webpage.
+            ix: Index of the block.
+            html: The HTML content of the webpage.
+
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        from .utils import aperform_completion_with_backoff
+
+        if self.verbose:
+            print(f"[LOG] Call LLM for {url} - block index: {ix}")
+
+        variable_values = {
+            "URL": url,
+            "HTML": escape_json_string(sanitize_html(html)),
+        }
+
+        prompt_with_variables = PROMPT_EXTRACT_BLOCKS
+        if self.instruction:
+            variable_values["REQUEST"] = self.instruction
+            prompt_with_variables = PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION
+
+        if self.extract_type == "schema" and self.schema:
+            variable_values["SCHEMA"] = json.dumps(self.schema, indent=2)
+            prompt_with_variables = PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
+
+        if self.extract_type == "schema" and not self.schema:
+            prompt_with_variables = PROMPT_EXTRACT_INFERRED_SCHEMA
+
+        for variable in variable_values:
+            prompt_with_variables = prompt_with_variables.replace(
+                "{" + variable + "}", variable_values[variable]
+            )
+
+        try:
+            response = await aperform_completion_with_backoff(
+                self.llm_config.provider,
+                prompt_with_variables,
+                self.llm_config.api_token,
+                base_url=self.llm_config.base_url,
+                json_response=self.force_json_response,
+                extra_args=self.extra_args,
+                base_delay=self.llm_config.backoff_base_delay,
+                max_attempts=self.llm_config.backoff_max_attempts,
+                exponential_factor=self.llm_config.backoff_exponential_factor
+            )
+            # Track usage
+            usage = TokenUsage(
+                completion_tokens=response.usage.completion_tokens,
+                prompt_tokens=response.usage.prompt_tokens,
+                total_tokens=response.usage.total_tokens,
+                completion_tokens_details=response.usage.completion_tokens_details.__dict__
+                if response.usage.completion_tokens_details
+                else {},
+                prompt_tokens_details=response.usage.prompt_tokens_details.__dict__
+                if response.usage.prompt_tokens_details
+                else {},
+            )
+            self.usages.append(usage)
+
+            # Update totals
+            self.total_usage.completion_tokens += usage.completion_tokens
+            self.total_usage.prompt_tokens += usage.prompt_tokens
+            self.total_usage.total_tokens += usage.total_tokens
+
+            try:
+                content = response.choices[0].message.content
+                blocks = None
+
+                if self.force_json_response:
+                    blocks = json.loads(content)
+                    if isinstance(blocks, dict):
+                        if len(blocks) == 1 and isinstance(list(blocks.values())[0], list):
+                            blocks = list(blocks.values())[0]
+                        else:
+                            blocks = [blocks]
+                    elif isinstance(blocks, list):
+                        blocks = blocks
+                else:
+                    blocks = extract_xml_data(["blocks"], content)["blocks"]
+                    blocks = json.loads(blocks)
+
+                for block in blocks:
+                    block["error"] = False
+            except Exception:
+                parsed, unparsed = split_and_parse_json_objects(
+                    response.choices[0].message.content
+                )
+                blocks = parsed
+                if unparsed:
+                    blocks.append(
+                        {"index": 0, "error": True, "tags": ["error"], "content": unparsed}
+                    )
+
+            if self.verbose:
+                print(
+                    "[LOG] Extracted",
+                    len(blocks),
+                    "blocks from URL:",
+                    url,
+                    "block index:",
+                    ix,
+                )
+            return blocks
+        except Exception as e:
+            if self.verbose:
+                print(f"[LOG] Error in LLM extraction: {e}")
+            return [
+                {
+                    "index": ix,
+                    "error": True,
+                    "tags": ["error"],
+                    "content": str(e),
+                }
+            ]
+
+    async def arun(self, url: str, sections: List[str]) -> List[Dict[str, Any]]:
+        """
+        Async version: Process sections with true parallelism using asyncio.gather.
+
+        Args:
+            url: The URL of the webpage.
+            sections: List of sections (strings) to process.
+
+        Returns:
+            A list of extracted blocks or chunks.
+        """
+        import asyncio
+
+        merged_sections = self._merge(
+            sections,
+            self.chunk_token_threshold,
+            overlap=int(self.chunk_token_threshold * self.overlap_rate),
+        )
+
+        extracted_content = []
+
+        # Create tasks for all sections to run in parallel
+        tasks = [
+            self.aextract(url, ix, sanitize_input_encode(section))
+            for ix, section in enumerate(merged_sections)
+        ]
+
+        # Execute all tasks concurrently
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Process results
+        for result in results:
+            if isinstance(result, Exception):
+                if self.verbose:
+                    print(f"Error in async extraction: {result}")
+                extracted_content.append(
+                    {
+                        "index": 0,
+                        "error": True,
+                        "tags": ["error"],
+                        "content": str(result),
+                    }
+                )
+            else:
+                extracted_content.extend(result)
+
+        return extracted_content
+
     def show_usage(self) -> None:
         """Print a detailed token usage report showing total and per-request usage."""
         print("\n=== Token Usage Summary ===")
@@ -1086,44 +1277,18 @@ class JsonElementExtractionStrategy(ExtractionStrategy):
     }
 
     @staticmethod
-    def generate_schema(
-        html: str,
-        schema_type: str = "CSS", # or XPATH
-        query: str = None,
-        target_json_example: str = None,
-        llm_config: 'LLMConfig' = create_llm_config(),
-        provider: str = None,
-        api_token: str = None,
-        **kwargs
-    ) -> dict:
+    def _build_schema_prompt(html: str, schema_type: str, query: str = None, target_json_example: str = None) -> str:
         """
-        Generate extraction schema from HTML content and optional query.
-        
-        Args:
-            html (str): The HTML content to analyze
-            query (str, optional): Natural language description of what data to extract
-            provider (str): Legacy Parameter. LLM provider to use 
-            api_token (str): Legacy Parameter. API token for LLM provider
-            llm_config (LLMConfig): LLM configuration object
-            prompt (str, optional): Custom prompt template to use
-            **kwargs: Additional args passed to LLM processor
-            
+        Build the prompt for schema generation. Shared by sync and async methods.
+
         Returns:
-            dict: Generated schema following the JsonElementExtractionStrategy format
+            str: Combined system and user prompt
         """
         from .prompts import JSON_SCHEMA_BUILDER
-        from .utils import perform_completion_with_backoff
-        for name, message in JsonElementExtractionStrategy._GENERATE_SCHEMA_UNWANTED_PROPS.items():
-            if locals()[name] is not None:
-                raise AttributeError(f"Setting '{name}' is deprecated. {message}")
-        
-        # Use default or custom prompt
+
         prompt_template = JSON_SCHEMA_BUILDER if schema_type == "CSS" else JSON_SCHEMA_BUILDER_XPATH
-        
-        # Build the prompt
-        system_message = {
-            "role": "system", 
-            "content": f"""You specialize in generating special JSON schemas for web scraping. This schema uses CSS or XPATH selectors to present a repetitive pattern in crawled HTML, such as a product in a product list or a search result item in a list of search results. We use this JSON schema to pass to a language model along with the HTML content to extract structured data from the HTML. The language model uses the JSON schema to extract data from the HTML and retrieve values for fields in the JSON schema, following the schema.
+
+        system_content = f"""You specialize in generating special JSON schemas for web scraping. This schema uses CSS or XPATH selectors to present a repetitive pattern in crawled HTML, such as a product in a product list or a search result item in a list of search results. We use this JSON schema to pass to a language model along with the HTML content to extract structured data from the HTML. The language model uses the JSON schema to extract data from the HTML and retrieve values for fields in the JSON schema, following the schema.
 
 Generating this HTML manually is not feasible, so you need to generate the JSON schema using the HTML content. The HTML copied from the crawled website is provided below, which we believe contains the repetitive pattern.
 
@@ -1144,31 +1309,27 @@ In this scenario, use your best judgment to generate the schema. You need to exa
 
 # What are the instructions and details for this schema generation?
 {prompt_template}"""
-        }
-        
-        user_message = {
-            "role": "user",
-            "content": f"""
+
+        user_content = f"""
                 HTML to analyze:
                 ```html
                 {html}
                 ```
                 """
-        }
 
         if query:
-            user_message["content"] += f"\n\n## Query or explanation of target/goal data item:\n{query}"
+            user_content += f"\n\n## Query or explanation of target/goal data item:\n{query}"
         if target_json_example:
-            user_message["content"] += f"\n\n## Example of target JSON object:\n```json\n{target_json_example}\n```"
+            user_content += f"\n\n## Example of target JSON object:\n```json\n{target_json_example}\n```"
 
         if query and not target_json_example:
-            user_message["content"] += """IMPORTANT: To remind you, in this process, we are not providing a rigid example of the adjacent objects we seek. We rely on your understanding of the explanation provided in the above section. Make sure to grasp what we are looking for and, based on that, create the best schema.."""
+            user_content += """IMPORTANT: To remind you, in this process, we are not providing a rigid example of the adjacent objects we seek. We rely on your understanding of the explanation provided in the above section. Make sure to grasp what we are looking for and, based on that, create the best schema.."""
         elif not query and target_json_example:
-            user_message["content"] += """IMPORTANT: Please remember that in this process, we provided a proper example of a target JSON object. Make sure to adhere to the structure and create a schema that exactly fits this example. If you find that some elements on the page do not match completely, vote for the majority."""
+            user_content += """IMPORTANT: Please remember that in this process, we provided a proper example of a target JSON object. Make sure to adhere to the structure and create a schema that exactly fits this example. If you find that some elements on the page do not match completely, vote for the majority."""
         elif not query and not target_json_example:
-            user_message["content"] += """IMPORTANT: Since we neither have a query nor an example, it is crucial to rely solely on the HTML content provided. Leverage your expertise to determine the schema based on the repetitive patterns observed in the content."""
-        
-        user_message["content"] += """IMPORTANT: 
+            user_content += """IMPORTANT: Since we neither have a query nor an example, it is crucial to rely solely on the HTML content provided. Leverage your expertise to determine the schema based on the repetitive patterns observed in the content."""
+
+        user_content += """IMPORTANT:
         0/ Ensure your schema remains reliable by avoiding selectors that appear to generate dynamically and are not dependable. You want a reliable schema, as it consistently returns the same data even after many page reloads.
         1/ DO NOT USE use base64 kind of classes, they are temporary and not reliable.
         2/ Every selector must refer to only one unique element. You should ensure your selector points to a single element and is unique to the place that contains the information. You have to use available techniques based on CSS or XPATH requested schema to make sure your selector is unique and also not fragile, meaning if we reload the page now or in the future, the selector should remain reliable.
@@ -1177,20 +1338,98 @@ In this scenario, use your best judgment to generate the schema. You need to exa
         Analyze the HTML and generate a JSON schema that follows the specified format. Only output valid JSON schema, nothing else.
         """
 
+        return "\n\n".join([system_content, user_content])
+
+    @staticmethod
+    def generate_schema(
+        html: str,
+        schema_type: str = "CSS",
+        query: str = None,
+        target_json_example: str = None,
+        llm_config: 'LLMConfig' = create_llm_config(),
+        provider: str = None,
+        api_token: str = None,
+        **kwargs
+    ) -> dict:
+        """
+        Generate extraction schema from HTML content and optional query (sync version).
+
+        Args:
+            html (str): The HTML content to analyze
+            query (str, optional): Natural language description of what data to extract
+            provider (str): Legacy Parameter. LLM provider to use
+            api_token (str): Legacy Parameter. API token for LLM provider
+            llm_config (LLMConfig): LLM configuration object
+            **kwargs: Additional args passed to LLM processor
+
+        Returns:
+            dict: Generated schema following the JsonElementExtractionStrategy format
+        """
+        from .utils import perform_completion_with_backoff
+
+        for name, message in JsonElementExtractionStrategy._GENERATE_SCHEMA_UNWANTED_PROPS.items():
+            if locals()[name] is not None:
+                raise AttributeError(f"Setting '{name}' is deprecated. {message}")
+
+        prompt = JsonElementExtractionStrategy._build_schema_prompt(html, schema_type, query, target_json_example)
+
         try:
-            # Call LLM with backoff handling
             response = perform_completion_with_backoff(
                 provider=llm_config.provider,
-                prompt_with_variables="\n\n".join([system_message["content"], user_message["content"]]),
-                json_response = True,                
+                prompt_with_variables=prompt,
+                json_response=True,
+                api_token=llm_config.api_token,
+                base_url=llm_config.base_url,
+                extra_args=kwargs
+            )
+            return json.loads(response.choices[0].message.content)
+        except Exception as e:
+            raise Exception(f"Failed to generate schema: {str(e)}")
+
+    @staticmethod
+    async def agenerate_schema(
+        html: str,
+        schema_type: str = "CSS",
+        query: str = None,
+        target_json_example: str = None,
+        llm_config: 'LLMConfig' = None,
+        **kwargs
+    ) -> dict:
+        """
+        Generate extraction schema from HTML content (async version).
+
+        Use this method when calling from async contexts (e.g., FastAPI) to avoid
+        issues with certain LLM providers (e.g., Gemini/Vertex AI) that require
+        async execution.
+
+        Args:
+            html (str): The HTML content to analyze
+            schema_type (str): "CSS" or "XPATH"
+            query (str, optional): Natural language description of what data to extract
+            target_json_example (str, optional): Example of desired JSON output
+            llm_config (LLMConfig): LLM configuration object
+            **kwargs: Additional args passed to LLM processor
+
+        Returns:
+            dict: Generated schema following the JsonElementExtractionStrategy format
+        """
+        from .utils import aperform_completion_with_backoff
+
+        if llm_config is None:
+            llm_config = create_llm_config()
+
+        prompt = JsonElementExtractionStrategy._build_schema_prompt(html, schema_type, query, target_json_example)
+
+        try:
+            response = await aperform_completion_with_backoff(
+                provider=llm_config.provider,
+                prompt_with_variables=prompt,
+                json_response=True,
                 api_token=llm_config.api_token,
                 base_url=llm_config.base_url,
                 extra_args=kwargs
             )
-            
-            # Extract and return schema
             return json.loads(response.choices[0].message.content)
-            
         except Exception as e:
             raise Exception(f"Failed to generate schema: {str(e)}")
 

crawl4ai/models.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, HttpUrl, PrivateAttr, Field
+from pydantic import BaseModel, HttpUrl, PrivateAttr, Field, ConfigDict
 from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
 from typing import AsyncGenerator
 from typing import Generic, TypeVar
@@ -152,9 +152,12 @@ class CrawlResult(BaseModel):
     network_requests: Optional[List[Dict[str, Any]]] = None
     console_messages: Optional[List[Dict[str, Any]]] = None
     tables: List[Dict] = Field(default_factory=list)  # NEW – [{headers,rows,caption,summary}]
+    # Cache validation metadata (Smart Cache)
+    head_fingerprint: Optional[str] = None
+    cached_at: Optional[float] = None
+    cache_status: Optional[str] = None  # "hit", "hit_validated", "hit_fallback", "miss"
 
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
 # NOTE: The StringCompatibleMarkdown class, custom __init__ method, property getters/setters,
 # and model_dump override all exist to support a smooth transition from markdown as a string
@@ -332,8 +335,7 @@ class AsyncCrawlResponse(BaseModel):
     network_requests: Optional[List[Dict[str, Any]]] = None
     console_messages: Optional[List[Dict[str, Any]]] = None
 
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
 ###############################
 # Scraping Models

crawl4ai/processors/pdf/processor.py

@@ -15,9 +15,9 @@ from .utils import (
     clean_pdf_text_to_html,
 )
 
-# Remove direct PyPDF2 imports from the top
-# import PyPDF2
-# from PyPDF2 import PdfReader
+# Remove direct pypdf imports from the top
+# import pypdf
+# from pypdf import PdfReader
 
 logger = logging.getLogger(__name__)
 
@@ -59,9 +59,9 @@ class NaivePDFProcessorStrategy(PDFProcessorStrategy):
                  save_images_locally: bool = False, image_save_dir: Optional[Path] = None, batch_size: int = 4):
         # Import check at initialization time
         try:
-            import PyPDF2
+            import pypdf
         except ImportError:
-            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            raise ImportError("pypdf is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
             
         self.image_dpi = image_dpi
         self.image_quality = image_quality
@@ -75,9 +75,9 @@ class NaivePDFProcessorStrategy(PDFProcessorStrategy):
     def process(self, pdf_path: Path) -> PDFProcessResult:
         # Import inside method to allow dependency to be optional
         try:
-            from PyPDF2 import PdfReader
+            from pypdf import PdfReader
         except ImportError:
-            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            raise ImportError("pypdf is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
             
         start_time = time()
         result = PDFProcessResult(
@@ -125,15 +125,15 @@ class NaivePDFProcessorStrategy(PDFProcessorStrategy):
         """Like process() but processes PDF pages in parallel batches"""
         # Import inside method to allow dependency to be optional
         try:
-            from PyPDF2 import PdfReader
-            import PyPDF2  # For type checking
+            from pypdf import PdfReader
+            import pypdf  # For type checking
         except ImportError:
-            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            raise ImportError("pypdf is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
             
         import concurrent.futures
         import threading
         
-        # Initialize PyPDF2 thread support
+        # Initialize pypdf thread support
         if not hasattr(threading.current_thread(), "_children"): 
             threading.current_thread()._children = set()
         
@@ -232,11 +232,11 @@ class NaivePDFProcessorStrategy(PDFProcessorStrategy):
         return pdf_page
 
     def _extract_images(self, page, image_dir: Optional[Path]) -> List[Dict]:
-        # Import PyPDF2 for type checking only when needed
+        # Import pypdf for type checking only when needed
         try:
-            import PyPDF2
+            from pypdf.generic import IndirectObject
         except ImportError:
-            raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+            raise ImportError("pypdf is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
             
         if not self.extract_images:
             return []
@@ -266,7 +266,7 @@ class NaivePDFProcessorStrategy(PDFProcessorStrategy):
                                     width = xobj.get('/Width', 0)
                                     height = xobj.get('/Height', 0)
                                     color_space = xobj.get('/ColorSpace', '/DeviceRGB')
-                                    if isinstance(color_space, PyPDF2.generic.IndirectObject):
+                                    if isinstance(color_space, IndirectObject):
                                         color_space = color_space.get_object()
 
                                     # Handle different image encodings
@@ -277,7 +277,7 @@ class NaivePDFProcessorStrategy(PDFProcessorStrategy):
                                     if '/FlateDecode' in filters:
                                         try:
                                             decode_parms = xobj.get('/DecodeParms', {})
-                                            if isinstance(decode_parms, PyPDF2.generic.IndirectObject):
+                                            if isinstance(decode_parms, IndirectObject):
                                                 decode_parms = decode_parms.get_object()
                                             
                                             predictor = decode_parms.get('/Predictor', 1)
@@ -416,10 +416,10 @@ class NaivePDFProcessorStrategy(PDFProcessorStrategy):
         # Import inside method to allow dependency to be optional 
         if reader is None:
             try:
-                from PyPDF2 import PdfReader
+                from pypdf import PdfReader
                 reader = PdfReader(pdf_path)
             except ImportError:
-                raise ImportError("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+                raise ImportError("pypdf is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
 
         meta = reader.metadata or {}
         created = self._parse_pdf_date(meta.get('/CreationDate', ''))
@@ -459,11 +459,11 @@ if __name__ == "__main__":
     from pathlib import Path
     
     try:
-        # Import PyPDF2 only when running the file directly
-        import PyPDF2
-        from PyPDF2 import PdfReader
+        # Import pypdf only when running the file directly
+        import pypdf
+        from pypdf import PdfReader
     except ImportError:
-        print("PyPDF2 is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
+        print("pypdf is required for PDF processing. Install with 'pip install crawl4ai[pdf]'")
         exit(1)
         
     current_dir = Path(__file__).resolve().parent

crawl4ai/proxy_strategy.py

@@ -1,7 +1,9 @@
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Tuple
 from abc import ABC, abstractmethod
 from itertools import cycle
 import os
+import asyncio
+import time
 
 
 ########### ATTENTION PEOPLE OF EARTH ###########
@@ -120,7 +122,7 @@ class ProxyConfig:
 
 class ProxyRotationStrategy(ABC):
     """Base abstract class for proxy rotation strategies"""
-    
+
     @abstractmethod
     async def get_next_proxy(self) -> Optional[ProxyConfig]:
         """Get next proxy configuration from the strategy"""
@@ -131,18 +133,81 @@ class ProxyRotationStrategy(ABC):
         """Add proxy configurations to the strategy"""
         pass
 
-class RoundRobinProxyStrategy:
-    """Simple round-robin proxy rotation strategy using ProxyConfig objects"""
+    @abstractmethod
+    async def get_proxy_for_session(
+        self,
+        session_id: str,
+        ttl: Optional[int] = None
+    ) -> Optional[ProxyConfig]:
+        """
+        Get or create a sticky proxy for a session.
+
+        If session_id already has an assigned proxy (and hasn't expired), return it.
+        If session_id is new, acquire a new proxy and associate it.
+
+        Args:
+            session_id: Unique session identifier
+            ttl: Optional time-to-live in seconds for this session
+
+        Returns:
+            ProxyConfig for this session
+        """
+        pass
+
+    @abstractmethod
+    async def release_session(self, session_id: str) -> None:
+        """
+        Release a sticky session, making the proxy available for reuse.
+
+        Args:
+            session_id: Session to release
+        """
+        pass
+
+    @abstractmethod
+    def get_session_proxy(self, session_id: str) -> Optional[ProxyConfig]:
+        """
+        Get the proxy for an existing session without creating new one.
+
+        Args:
+            session_id: Session to look up
+
+        Returns:
+            ProxyConfig if session exists and hasn't expired, None otherwise
+        """
+        pass
+
+    @abstractmethod
+    def get_active_sessions(self) -> Dict[str, ProxyConfig]:
+        """
+        Get all active sticky sessions.
+
+        Returns:
+            Dictionary mapping session_id to ProxyConfig
+        """
+        pass
+
+class RoundRobinProxyStrategy(ProxyRotationStrategy):
+    """Simple round-robin proxy rotation strategy using ProxyConfig objects.
+
+    Supports sticky sessions where a session_id can be bound to a specific proxy
+    for the duration of the session. This is useful for deep crawling where
+    you want to maintain the same IP address across multiple requests.
+    """
 
     def __init__(self, proxies: List[ProxyConfig] = None):
         """
         Initialize with optional list of proxy configurations
-        
+
         Args:
             proxies: List of ProxyConfig objects
         """
-        self._proxies = []
+        self._proxies: List[ProxyConfig] = []
         self._proxy_cycle = None
+        # Session tracking: maps session_id -> (ProxyConfig, created_at, ttl)
+        self._sessions: Dict[str, Tuple[ProxyConfig, float, Optional[int]]] = {}
+        self._session_lock = asyncio.Lock()
+
         if proxies:
             self.add_proxies(proxies)
 
@@ -156,3 +221,121 @@ class RoundRobinProxyStrategy:
         if not self._proxy_cycle:
             return None
         return next(self._proxy_cycle)
+
+    async def get_proxy_for_session(
+        self,
+        session_id: str,
+        ttl: Optional[int] = None
+    ) -> Optional[ProxyConfig]:
+        """
+        Get or create a sticky proxy for a session.
+
+        If session_id already has an assigned proxy (and hasn't expired), return it.
+        If session_id is new, acquire a new proxy and associate it.
+
+        Args:
+            session_id: Unique session identifier
+            ttl: Optional time-to-live in seconds for this session
+
+        Returns:
+            ProxyConfig for this session
+        """
+        async with self._session_lock:
+            # Check if session exists and hasn't expired
+            if session_id in self._sessions:
+                proxy, created_at, session_ttl = self._sessions[session_id]
+
+                # Check TTL expiration
+                effective_ttl = ttl if ttl is not None else session_ttl
+                if effective_ttl is not None:
+                    elapsed = time.time() - created_at
+                    if elapsed >= effective_ttl:
+                        # Session expired, remove it and get new proxy
+                        del self._sessions[session_id]
+                    else:
+                        return proxy
+                else:
+                    return proxy
+
+            # Acquire new proxy for this session
+            proxy = await self.get_next_proxy()
+            if proxy:
+                self._sessions[session_id] = (proxy, time.time(), ttl)
+
+            return proxy
+
+    async def release_session(self, session_id: str) -> None:
+        """
+        Release a sticky session, making the proxy available for reuse.
+
+        Args:
+            session_id: Session to release
+        """
+        async with self._session_lock:
+            if session_id in self._sessions:
+                del self._sessions[session_id]
+
+    def get_session_proxy(self, session_id: str) -> Optional[ProxyConfig]:
+        """
+        Get the proxy for an existing session without creating new one.
+
+        Args:
+            session_id: Session to look up
+
+        Returns:
+            ProxyConfig if session exists and hasn't expired, None otherwise
+        """
+        if session_id not in self._sessions:
+            return None
+
+        proxy, created_at, ttl = self._sessions[session_id]
+
+        # Check TTL expiration
+        if ttl is not None:
+            elapsed = time.time() - created_at
+            if elapsed >= ttl:
+                return None
+
+        return proxy
+
+    def get_active_sessions(self) -> Dict[str, ProxyConfig]:
+        """
+        Get all active sticky sessions (excluding expired ones).
+
+        Returns:
+            Dictionary mapping session_id to ProxyConfig
+        """
+        current_time = time.time()
+        active_sessions = {}
+
+        for session_id, (proxy, created_at, ttl) in self._sessions.items():
+            # Skip expired sessions
+            if ttl is not None:
+                elapsed = current_time - created_at
+                if elapsed >= ttl:
+                    continue
+            active_sessions[session_id] = proxy
+
+        return active_sessions
+
+    async def cleanup_expired_sessions(self) -> int:
+        """
+        Remove all expired sessions from tracking.
+
+        Returns:
+            Number of sessions removed
+        """
+        async with self._session_lock:
+            current_time = time.time()
+            expired = []
+
+            for session_id, (proxy, created_at, ttl) in self._sessions.items():
+                if ttl is not None:
+                    elapsed = current_time - created_at
+                    if elapsed >= ttl:
+                        expired.append(session_id)
+
+            for session_id in expired:
+                del self._sessions[session_id]
+
+            return len(expired)

crawl4ai/table_extraction.py

@@ -795,6 +795,9 @@ Return only a JSON array of extracted tables following the specified format."""
                     api_token=self.llm_config.api_token,
                     base_url=self.llm_config.base_url,
                     json_response=True,
+                    base_delay=self.llm_config.backoff_base_delay,
+                    max_attempts=self.llm_config.backoff_max_attempts,
+                    exponential_factor=self.llm_config.backoff_exponential_factor,
                     extra_args=self.extra_args
                 )
                 
@@ -1116,6 +1119,9 @@ Return only a JSON array of extracted tables following the specified format."""
                     api_token=self.llm_config.api_token,
                     base_url=self.llm_config.base_url,
                     json_response=True,
+                    base_delay=self.llm_config.backoff_base_delay,
+                    max_attempts=self.llm_config.backoff_max_attempts,
+                    exponential_factor=self.llm_config.backoff_exponential_factor,
                     extra_args=self.extra_args
                 )
                 

crawl4ai/utils.py

@@ -47,6 +47,7 @@ from urllib.parse import (
     urljoin, urlparse, urlunparse,
     parse_qsl, urlencode, quote, unquote
 )
+import inspect
 
 
 # Monkey patch to fix wildcard handling in urllib.robotparser
@@ -1744,6 +1745,9 @@ def perform_completion_with_backoff(
     api_token,
     json_response=False,
     base_url=None,
+    base_delay=2,
+    max_attempts=3,
+    exponential_factor=2,
     **kwargs,
 ):
     """
@@ -1760,6 +1764,9 @@ def perform_completion_with_backoff(
         api_token (str): The API token for authentication.
         json_response (bool): Whether to request a JSON response. Defaults to False.
         base_url (Optional[str]): The base URL for the API. Defaults to None.
+        base_delay (int): The base delay in seconds. Defaults to 2.
+        max_attempts (int): The maximum number of attempts. Defaults to 3.
+        exponential_factor (int): The exponential factor. Defaults to 2.
         **kwargs: Additional arguments for the API request.
 
     Returns:
@@ -1768,9 +1775,8 @@ def perform_completion_with_backoff(
 
     from litellm import completion
     from litellm.exceptions import RateLimitError
-
-    max_attempts = 3
-    base_delay = 2  # Base delay in seconds, you can adjust this based on your needs
+    import litellm
+    litellm.drop_params = True  # Auto-drop unsupported params (e.g., temperature for O-series/GPT-5)
 
     extra_args = {"temperature": 0.01, "api_key": api_token, "base_url": base_url}
     if json_response:
@@ -1797,7 +1803,7 @@ def perform_completion_with_backoff(
             # Check if we have exhausted our max attempts
             if attempt < max_attempts - 1:
                 # Calculate the delay and wait
-                delay = base_delay * (2**attempt)  # Exponential backoff formula
+                delay = base_delay * (exponential_factor**attempt)  # Exponential backoff formula
                 print(f"Waiting for {delay} seconds before retrying...")
                 time.sleep(delay)
             else:
@@ -1824,6 +1830,87 @@ def perform_completion_with_backoff(
             # ]
 
 
+async def aperform_completion_with_backoff(
+    provider,
+    prompt_with_variables,
+    api_token,
+    json_response=False,
+    base_url=None,
+    base_delay=2,
+    max_attempts=3,
+    exponential_factor=2,
+    **kwargs,
+):
+    """
+    Async version: Perform an API completion request with exponential backoff.
+
+    How it works:
+    1. Sends an async completion request to the API.
+    2. Retries on rate-limit errors with exponential delays (async).
+    3. Returns the API response or an error after all retries.
+
+    Args:
+        provider (str): The name of the API provider.
+        prompt_with_variables (str): The input prompt for the completion request.
+        api_token (str): The API token for authentication.
+        json_response (bool): Whether to request a JSON response. Defaults to False.
+        base_url (Optional[str]): The base URL for the API. Defaults to None.
+        base_delay (int): The base delay in seconds. Defaults to 2.
+        max_attempts (int): The maximum number of attempts. Defaults to 3.
+        exponential_factor (int): The exponential factor. Defaults to 2.
+        **kwargs: Additional arguments for the API request.
+
+    Returns:
+        dict: The API response or an error message after all retries.
+    """
+
+    from litellm import acompletion
+    from litellm.exceptions import RateLimitError
+    import litellm
+    import asyncio
+    litellm.drop_params = True  # Auto-drop unsupported params (e.g., temperature for O-series/GPT-5)
+
+    extra_args = {"temperature": 0.01, "api_key": api_token, "base_url": base_url}
+    if json_response:
+        extra_args["response_format"] = {"type": "json_object"}
+
+    if kwargs.get("extra_args"):
+        extra_args.update(kwargs["extra_args"])
+
+    for attempt in range(max_attempts):
+        try:
+            response = await acompletion(
+                model=provider,
+                messages=[{"role": "user", "content": prompt_with_variables}],
+                **extra_args,
+            )
+            return response  # Return the successful response
+        except RateLimitError as e:
+            print("Rate limit error:", str(e))
+
+            if attempt == max_attempts - 1:
+                # Last attempt failed, raise the error.
+                raise
+
+            # Check if we have exhausted our max attempts
+            if attempt < max_attempts - 1:
+                # Calculate the delay and wait
+                delay = base_delay * (exponential_factor**attempt)  # Exponential backoff formula
+                print(f"Waiting for {delay} seconds before retrying...")
+                await asyncio.sleep(delay)
+            else:
+                # Return an error response after exhausting all retries
+                return [
+                    {
+                        "index": 0,
+                        "tags": ["error"],
+                        "content": ["Rate limit error. Please try again later."],
+                    }
+                ]
+        except Exception as e:
+            raise e  # Raise any other exceptions immediately
+
+
 def extract_blocks(url, html, provider=DEFAULT_PROVIDER, api_token=None, base_url=None):
     """
     Extract content blocks from website HTML using an AI provider.
@@ -2378,6 +2465,54 @@ def normalize_url_tmp(href, base_url):
     return href.strip()
 
 
+def quick_extract_links(html: str, base_url: str) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Fast link extraction for prefetch mode.
+    Only extracts <a href> tags - no media, no cleaning, no heavy processing.
+
+    Args:
+        html: Raw HTML string
+        base_url: Base URL for resolving relative links
+
+    Returns:
+        {"internal": [{"href": "...", "text": "..."}], "external": [...]}
+    """
+    from lxml.html import document_fromstring
+
+    try:
+        doc = document_fromstring(html)
+    except Exception:
+        return {"internal": [], "external": []}
+
+    base_domain = get_base_domain(base_url)
+    internal: List[Dict[str, str]] = []
+    external: List[Dict[str, str]] = []
+    seen: Set[str] = set()
+
+    for a in doc.xpath("//a[@href]"):
+        href = a.get("href", "").strip()
+        if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+            continue
+
+        # Normalize URL
+        normalized = normalize_url_for_deep_crawl(href, base_url)
+        if not normalized or normalized in seen:
+            continue
+        seen.add(normalized)
+
+        # Extract text (truncated for memory efficiency)
+        text = (a.text_content() or "").strip()[:200]
+
+        link_data = {"href": normalized, "text": text}
+
+        if is_external_url(normalized, base_domain):
+            external.append(link_data)
+        else:
+            internal.append(link_data)
+
+    return {"internal": internal, "external": external}
+
+
 def get_base_domain(url: str) -> str:
     """
     Extract the base domain from a given URL, handling common edge cases.
@@ -2745,6 +2880,67 @@ def generate_content_hash(content: str) -> str:
     # return hashlib.sha256(content.encode()).hexdigest()
 
 
+def compute_head_fingerprint(head_html: str) -> str:
+    """
+    Compute a fingerprint of <head> content for cache validation.
+
+    Focuses on content that typically changes when page updates:
+    - <title>
+    - <meta name="description">
+    - <meta property="og:title|og:description|og:image|og:updated_time">
+    - <meta property="article:modified_time">
+    - <meta name="last-modified">
+
+    Uses xxhash for speed, combines multiple signals into a single hash.
+
+    Args:
+        head_html: The HTML content of the <head> section
+
+    Returns:
+        A hex string fingerprint, or empty string if no signals found
+    """
+    if not head_html:
+        return ""
+
+    head_lower = head_html.lower()
+    signals = []
+
+    # Extract title
+    title_match = re.search(r'<title[^>]*>(.*?)</title>', head_lower, re.DOTALL)
+    if title_match:
+        signals.append(title_match.group(1).strip())
+
+    # Meta tags to extract (name or property attribute, and the value to match)
+    meta_tags = [
+        ("name", "description"),
+        ("name", "last-modified"),
+        ("property", "og:title"),
+        ("property", "og:description"),
+        ("property", "og:image"),
+        ("property", "og:updated_time"),
+        ("property", "article:modified_time"),
+    ]
+
+    for attr_type, attr_value in meta_tags:
+        # Handle both attribute orders: attr="value" content="..." and content="..." attr="value"
+        patterns = [
+            rf'<meta[^>]*{attr_type}=["\']{ re.escape(attr_value)}["\'][^>]*content=["\']([^"\']*)["\']',
+            rf'<meta[^>]*content=["\']([^"\']*)["\'][^>]*{attr_type}=["\']{re.escape(attr_value)}["\']',
+        ]
+        for pattern in patterns:
+            match = re.search(pattern, head_lower)
+            if match:
+                signals.append(match.group(1).strip())
+                break  # Found this tag, move to next
+
+    if not signals:
+        return ""
+
+    # Combine signals and hash
+    combined = '|'.join(signals)
+    return xxhash.xxh64(combined.encode()).hexdigest()
+
+
 def ensure_content_dirs(base_path: str) -> Dict[str, str]:
     """Create content directories if they don't exist"""
     dirs = {
@@ -3529,4 +3725,52 @@ def get_memory_stats() -> Tuple[float, float, float]:
     available_gb = get_true_available_memory_gb()
     used_percent = get_true_memory_usage_percent()
     
-    return used_percent, available_gb, total_gb
+    return used_percent, available_gb, total_gb
+
+
+# Hook utilities for Docker API
+def hooks_to_string(hooks: Dict[str, Callable]) -> Dict[str, str]:
+    """
+    Convert hook function objects to string representations for Docker API.
+
+    This utility simplifies the process of using hooks with the Docker API by converting
+    Python function objects into the string format required by the API.
+
+    Args:
+        hooks: Dictionary mapping hook point names to Python function objects.
+               Functions should be async and follow hook signature requirements.
+
+    Returns:
+        Dictionary mapping hook point names to string representations of the functions.
+
+    Example:
+        >>> async def my_hook(page, context, **kwargs):
+        ...     await page.set_viewport_size({"width": 1920, "height": 1080})
+        ...     return page
+        >>>
+        >>> hooks_dict = {"on_page_context_created": my_hook}
+        >>> api_hooks = hooks_to_string(hooks_dict)
+        >>> # api_hooks is now ready to use with Docker API
+
+    Raises:
+        ValueError: If a hook is not callable or source cannot be extracted
+    """
+    result = {}
+
+    for hook_name, hook_func in hooks.items():
+        if not callable(hook_func):
+            raise ValueError(f"Hook '{hook_name}' must be a callable function, got {type(hook_func)}")
+
+        try:
+            # Get the source code of the function
+            source = inspect.getsource(hook_func)
+            # Remove any leading indentation to get clean source
+            source = textwrap.dedent(source)
+            result[hook_name] = source
+        except (OSError, TypeError) as e:
+            raise ValueError(
+                f"Cannot extract source code for hook '{hook_name}'. "
+                f"Make sure the function is defined in a file (not interactively). Error: {e}"
+            )
+
+    return result

deploy/docker/README.md

@@ -12,6 +12,7 @@
   - [Python SDK](#python-sdk)
   - [Understanding Request Schema](#understanding-request-schema)
   - [REST API Examples](#rest-api-examples)
+  - [Asynchronous Jobs with Webhooks](#asynchronous-jobs-with-webhooks)
 - [Additional API Endpoints](#additional-api-endpoints)
   - [HTML Extraction Endpoint](#html-extraction-endpoint)
   - [Screenshot Endpoint](#screenshot-endpoint)
@@ -58,15 +59,13 @@ Pull and run images directly from Docker Hub without building locally.
 
 #### 1. Pull the Image
 
-Our latest release candidate is `0.7.0-r1`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system.
-
-> ⚠️ **Important Note**: The `latest` tag currently points to the stable `0.6.0` version. After testing and validation, `0.7.0` (without -r1) will be released and `latest` will be updated. For now, please use `0.7.0-r1` to test the new features.
+Our latest stable release is `0.8.0`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system.
 
 ```bash
-# Pull the release candidate (for testing new features)
-docker pull unclecode/crawl4ai:0.7.0-r1
+# Pull the latest stable version (0.8.0)
+docker pull unclecode/crawl4ai:0.8.0
 
-# Or pull the current stable version (0.6.0)
+# Or use the latest tag (points to 0.8.0)
 docker pull unclecode/crawl4ai:latest
 ```
 
@@ -101,7 +100,7 @@ EOL
       -p 11235:11235 \
       --name crawl4ai \
       --shm-size=1g \
-      unclecode/crawl4ai:0.7.0-r1
+      unclecode/crawl4ai:0.8.0
     ```
 
 *   **With LLM support:**
@@ -112,7 +111,7 @@ EOL
       --name crawl4ai \
       --env-file .llm.env \
       --shm-size=1g \
-      unclecode/crawl4ai:0.7.0-r1
+      unclecode/crawl4ai:0.8.0
     ```
 
 > The server will be available at `http://localhost:11235`. Visit `/playground` to access the interactive testing interface.
@@ -185,7 +184,7 @@ The `docker-compose.yml` file in the project root provides a simplified approach
     ```bash
     # Pulls and runs the release candidate from Docker Hub
     # Automatically selects the correct architecture
-    IMAGE=unclecode/crawl4ai:0.7.0-r1 docker compose up -d
+    IMAGE=unclecode/crawl4ai:0.8.0 docker compose up -d
     ```
 
 *   **Build and Run Locally:**
@@ -648,6 +647,194 @@ async def test_stream_crawl(token: str = None): # Made token optional
 # asyncio.run(test_stream_crawl())
 ```
 
+### Asynchronous Jobs with Webhooks
+
+For long-running crawls or when you want to avoid keeping connections open, use the job queue endpoints. Instead of polling for results, configure a webhook to receive notifications when jobs complete.
+
+#### Why Use Jobs & Webhooks?
+
+- **No Polling Required** - Get notified when crawls complete instead of constantly checking status
+- **Better Resource Usage** - Free up client connections while jobs run in the background
+- **Scalable Architecture** - Ideal for high-volume crawling with TypeScript/Node.js clients or microservices
+- **Reliable Delivery** - Automatic retry with exponential backoff (5 attempts: 1s → 2s → 4s → 8s → 16s)
+
+#### How It Works
+
+1. **Submit Job** → POST to `/crawl/job` with optional `webhook_config`
+2. **Get Task ID** → Receive a `task_id` immediately
+3. **Job Runs** → Crawl executes in the background
+4. **Webhook Fired** → Server POSTs completion notification to your webhook URL
+5. **Fetch Results** → If data wasn't included in webhook, GET `/crawl/job/{task_id}`
+
+#### Quick Example
+
+```bash
+# Submit a crawl job with webhook notification
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"],
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
+      "webhook_data_in_payload": false
+    }
+  }'
+
+# Response: {"task_id": "crawl_a1b2c3d4"}
+```
+
+**Your webhook receives:**
+```json
+{
+  "task_id": "crawl_a1b2c3d4",
+  "task_type": "crawl",
+  "status": "completed",
+  "timestamp": "2025-10-21T10:30:00.000000+00:00",
+  "urls": ["https://example.com"]
+}
+```
+
+Then fetch the results:
+```bash
+curl http://localhost:11235/crawl/job/crawl_a1b2c3d4
+```
+
+#### Include Data in Webhook
+
+Set `webhook_data_in_payload: true` to receive the full crawl results directly in the webhook:
+
+```bash
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"],
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
+      "webhook_data_in_payload": true
+    }
+  }'
+```
+
+**Your webhook receives the complete data:**
+```json
+{
+  "task_id": "crawl_a1b2c3d4",
+  "task_type": "crawl",
+  "status": "completed",
+  "timestamp": "2025-10-21T10:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "data": {
+    "markdown": "...",
+    "html": "...",
+    "links": {...},
+    "metadata": {...}
+  }
+}
+```
+
+#### Webhook Authentication
+
+Add custom headers for authentication:
+
+```json
+{
+  "urls": ["https://example.com"],
+  "webhook_config": {
+    "webhook_url": "https://myapp.com/webhooks/crawl",
+    "webhook_data_in_payload": false,
+    "webhook_headers": {
+      "X-Webhook-Secret": "your-secret-token",
+      "X-Service-ID": "crawl4ai-prod"
+    }
+  }
+}
+```
+
+#### Global Default Webhook
+
+Configure a default webhook URL in `config.yml` for all jobs:
+
+```yaml
+webhooks:
+  enabled: true
+  default_url: "https://myapp.com/webhooks/default"
+  data_in_payload: false
+  retry:
+    max_attempts: 5
+    initial_delay_ms: 1000
+    max_delay_ms: 32000
+    timeout_ms: 30000
+```
+
+Now jobs without `webhook_config` automatically use the default webhook.
+
+#### Job Status Polling (Without Webhooks)
+
+If you prefer polling instead of webhooks, just omit `webhook_config`:
+
+```bash
+# Submit job
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{"urls": ["https://example.com"]}'
+# Response: {"task_id": "crawl_xyz"}
+
+# Poll for status
+curl http://localhost:11235/crawl/job/crawl_xyz
+```
+
+The response includes `status` field: `"processing"`, `"completed"`, or `"failed"`.
+
+#### LLM Extraction Jobs with Webhooks
+
+The same webhook system works for LLM extraction jobs via `/llm/job`:
+
+```bash
+# Submit LLM extraction job with webhook
+curl -X POST http://localhost:11235/llm/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "url": "https://example.com/article",
+    "q": "Extract the article title, author, and main points",
+    "provider": "openai/gpt-4o-mini",
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/llm-complete",
+      "webhook_data_in_payload": true,
+      "webhook_headers": {
+        "X-Webhook-Secret": "your-secret-token"
+      }
+    }
+  }'
+
+# Response: {"task_id": "llm_1234567890"}
+```
+
+**Your webhook receives:**
+```json
+{
+  "task_id": "llm_1234567890",
+  "task_type": "llm_extraction",
+  "status": "completed",
+  "timestamp": "2025-10-22T12:30:00.000000+00:00",
+  "urls": ["https://example.com/article"],
+  "data": {
+    "extracted_content": {
+      "title": "Understanding Web Scraping",
+      "author": "John Doe",
+      "main_points": ["Point 1", "Point 2", "Point 3"]
+    }
+  }
+}
+```
+
+**Key Differences for LLM Jobs:**
+- Task type is `"llm_extraction"` instead of `"crawl"`
+- Extracted data is in `data.extracted_content`
+- Single URL only (not an array)
+- Supports schema-based extraction with `schema` parameter
+
+> 💡 **Pro tip**: See [WEBHOOK_EXAMPLES.md](./WEBHOOK_EXAMPLES.md) for detailed examples including TypeScript client code, Flask webhook handlers, and failure handling.
+
 ---
 
 ## Metrics & Monitoring
@@ -826,10 +1013,11 @@ We're here to help you succeed with Crawl4AI! Here's how to get support:
 
 In this guide, we've covered everything you need to get started with Crawl4AI's Docker deployment:
 - Building and running the Docker container
-- Configuring the environment  
+- Configuring the environment
 - Using the interactive playground for testing
 - Making API requests with proper typing
 - Using the Python SDK
+- Asynchronous job queues with webhook notifications
 - Leveraging specialized endpoints for screenshots, PDFs, and JavaScript execution
 - Connecting via the Model Context Protocol (MCP)
 - Monitoring your deployment

deploy/docker/WEBHOOK_EXAMPLES.md

@@ -0,0 +1,378 @@
+# Webhook Feature Examples
+
+This document provides examples of how to use the webhook feature for crawl jobs in Crawl4AI.
+
+## Overview
+
+The webhook feature allows you to receive notifications when crawl jobs complete, eliminating the need for polling. Webhooks are sent with exponential backoff retry logic to ensure reliable delivery.
+
+## Configuration
+
+### Global Configuration (config.yml)
+
+You can configure default webhook settings in `config.yml`:
+
+```yaml
+webhooks:
+  enabled: true
+  default_url: null  # Optional: default webhook URL for all jobs
+  data_in_payload: false  # Optional: default behavior for including data
+  retry:
+    max_attempts: 5
+    initial_delay_ms: 1000  # 1s, 2s, 4s, 8s, 16s exponential backoff
+    max_delay_ms: 32000
+    timeout_ms: 30000  # 30s timeout per webhook call
+  headers:  # Optional: default headers to include
+    User-Agent: "Crawl4AI-Webhook/1.0"
+```
+
+## API Usage Examples
+
+### Example 1: Basic Webhook (Notification Only)
+
+Send a webhook notification without including the crawl data in the payload.
+
+**Request:**
+```bash
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"],
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
+      "webhook_data_in_payload": false
+    }
+  }'
+```
+
+**Response:**
+```json
+{
+  "task_id": "crawl_a1b2c3d4"
+}
+```
+
+**Webhook Payload Received:**
+```json
+{
+  "task_id": "crawl_a1b2c3d4",
+  "task_type": "crawl",
+  "status": "completed",
+  "timestamp": "2025-10-21T10:30:00.000000+00:00",
+  "urls": ["https://example.com"]
+}
+```
+
+Your webhook handler should then fetch the results:
+```bash
+curl http://localhost:11235/crawl/job/crawl_a1b2c3d4
+```
+
+### Example 2: Webhook with Data Included
+
+Include the full crawl results in the webhook payload.
+
+**Request:**
+```bash
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"],
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
+      "webhook_data_in_payload": true
+    }
+  }'
+```
+
+**Webhook Payload Received:**
+```json
+{
+  "task_id": "crawl_a1b2c3d4",
+  "task_type": "crawl",
+  "status": "completed",
+  "timestamp": "2025-10-21T10:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "data": {
+    "markdown": "...",
+    "html": "...",
+    "links": {...},
+    "metadata": {...}
+  }
+}
+```
+
+### Example 3: Webhook with Custom Headers
+
+Include custom headers for authentication or identification.
+
+**Request:**
+```bash
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"],
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
+      "webhook_data_in_payload": false,
+      "webhook_headers": {
+        "X-Webhook-Secret": "my-secret-token",
+        "X-Service-ID": "crawl4ai-production"
+      }
+    }
+  }'
+```
+
+The webhook will be sent with these additional headers plus the default headers from config.
+
+### Example 4: Failure Notification
+
+When a crawl job fails, a webhook is sent with error details.
+
+**Webhook Payload on Failure:**
+```json
+{
+  "task_id": "crawl_a1b2c3d4",
+  "task_type": "crawl",
+  "status": "failed",
+  "timestamp": "2025-10-21T10:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "error": "Connection timeout after 30s"
+}
+```
+
+### Example 5: Using Global Default Webhook
+
+If you set a `default_url` in config.yml, jobs without webhook_config will use it:
+
+**config.yml:**
+```yaml
+webhooks:
+  enabled: true
+  default_url: "https://myapp.com/webhooks/default"
+  data_in_payload: false
+```
+
+**Request (no webhook_config needed):**
+```bash
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"]
+  }'
+```
+
+The webhook will be sent to the default URL configured in config.yml.
+
+### Example 6: LLM Extraction Job with Webhook
+
+Use webhooks with the LLM extraction endpoint for asynchronous processing.
+
+**Request:**
+```bash
+curl -X POST http://localhost:11235/llm/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "url": "https://example.com/article",
+    "q": "Extract the article title, author, and publication date",
+    "schema": "{\"type\": \"object\", \"properties\": {\"title\": {\"type\": \"string\"}, \"author\": {\"type\": \"string\"}, \"date\": {\"type\": \"string\"}}}",
+    "cache": false,
+    "provider": "openai/gpt-4o-mini",
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/llm-complete",
+      "webhook_data_in_payload": true
+    }
+  }'
+```
+
+**Response:**
+```json
+{
+  "task_id": "llm_1698765432_12345"
+}
+```
+
+**Webhook Payload Received:**
+```json
+{
+  "task_id": "llm_1698765432_12345",
+  "task_type": "llm_extraction",
+  "status": "completed",
+  "timestamp": "2025-10-21T10:30:00.000000+00:00",
+  "urls": ["https://example.com/article"],
+  "data": {
+    "extracted_content": {
+      "title": "Understanding Web Scraping",
+      "author": "John Doe",
+      "date": "2025-10-21"
+    }
+  }
+}
+```
+
+## Webhook Handler Example
+
+Here's a simple Python Flask webhook handler that supports both crawl and LLM extraction jobs:
+
+```python
+from flask import Flask, request, jsonify
+import requests
+
+app = Flask(__name__)
+
+@app.route('/webhooks/crawl-complete', methods=['POST'])
+def handle_crawl_webhook():
+    payload = request.json
+
+    task_id = payload['task_id']
+    task_type = payload['task_type']
+    status = payload['status']
+
+    if status == 'completed':
+        # If data not in payload, fetch it
+        if 'data' not in payload:
+            # Determine endpoint based on task type
+            endpoint = 'crawl' if task_type == 'crawl' else 'llm'
+            response = requests.get(f'http://localhost:11235/{endpoint}/job/{task_id}')
+            data = response.json()
+        else:
+            data = payload['data']
+
+        # Process based on task type
+        if task_type == 'crawl':
+            print(f"Processing crawl results for {task_id}")
+            # Handle crawl results
+            results = data.get('results', [])
+            for result in results:
+                print(f"  - {result.get('url')}: {len(result.get('markdown', ''))} chars")
+
+        elif task_type == 'llm_extraction':
+            print(f"Processing LLM extraction for {task_id}")
+            # Handle LLM extraction
+            # Note: Webhook sends 'extracted_content', API returns 'result'
+            extracted = data.get('extracted_content', data.get('result', {}))
+            print(f"  - Extracted: {extracted}")
+
+        # Your business logic here...
+
+    elif status == 'failed':
+        error = payload.get('error', 'Unknown error')
+        print(f"{task_type} job {task_id} failed: {error}")
+        # Handle failure...
+
+    return jsonify({"status": "received"}), 200
+
+if __name__ == '__main__':
+    app.run(port=8080)
+```
+
+## Retry Logic
+
+The webhook delivery service uses exponential backoff retry logic:
+
+- **Attempts:** Up to 5 attempts by default
+- **Delays:** 1s → 2s → 4s → 8s → 16s
+- **Timeout:** 30 seconds per attempt
+- **Retry Conditions:**
+  - Server errors (5xx status codes)
+  - Network errors
+  - Timeouts
+- **No Retry:**
+  - Client errors (4xx status codes)
+  - Successful delivery (2xx status codes)
+
+## Benefits
+
+1. **No Polling Required** - Eliminates constant API calls to check job status
+2. **Real-time Notifications** - Immediate notification when jobs complete
+3. **Reliable Delivery** - Exponential backoff ensures webhooks are delivered
+4. **Flexible** - Choose between notification-only or full data delivery
+5. **Secure** - Support for custom headers for authentication
+6. **Configurable** - Global defaults or per-job configuration
+7. **Universal Support** - Works with both `/crawl/job` and `/llm/job` endpoints
+
+## TypeScript Client Example
+
+```typescript
+interface WebhookConfig {
+  webhook_url: string;
+  webhook_data_in_payload?: boolean;
+  webhook_headers?: Record<string, string>;
+}
+
+interface CrawlJobRequest {
+  urls: string[];
+  browser_config?: Record<string, any>;
+  crawler_config?: Record<string, any>;
+  webhook_config?: WebhookConfig;
+}
+
+interface LLMJobRequest {
+  url: string;
+  q: string;
+  schema?: string;
+  cache?: boolean;
+  provider?: string;
+  webhook_config?: WebhookConfig;
+}
+
+async function createCrawlJob(request: CrawlJobRequest) {
+  const response = await fetch('http://localhost:11235/crawl/job', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(request)
+  });
+
+  const { task_id } = await response.json();
+  return task_id;
+}
+
+async function createLLMJob(request: LLMJobRequest) {
+  const response = await fetch('http://localhost:11235/llm/job', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(request)
+  });
+
+  const { task_id } = await response.json();
+  return task_id;
+}
+
+// Usage - Crawl Job
+const crawlTaskId = await createCrawlJob({
+  urls: ['https://example.com'],
+  webhook_config: {
+    webhook_url: 'https://myapp.com/webhooks/crawl-complete',
+    webhook_data_in_payload: false,
+    webhook_headers: {
+      'X-Webhook-Secret': 'my-secret'
+    }
+  }
+});
+
+// Usage - LLM Extraction Job
+const llmTaskId = await createLLMJob({
+  url: 'https://example.com/article',
+  q: 'Extract the main points from this article',
+  provider: 'openai/gpt-4o-mini',
+  webhook_config: {
+    webhook_url: 'https://myapp.com/webhooks/llm-complete',
+    webhook_data_in_payload: true,
+    webhook_headers: {
+      'X-Webhook-Secret': 'my-secret'
+    }
+  }
+});
+```
+
+## Monitoring and Debugging
+
+Webhook delivery attempts are logged at INFO level:
+- Successful deliveries
+- Retry attempts with delays
+- Final failures after max attempts
+
+Check the application logs for webhook delivery status:
+```bash
+docker logs crawl4ai-container | grep -i webhook
+```

deploy/docker/api.py

@@ -46,6 +46,7 @@ from utils import (
     get_llm_temperature,
     get_llm_base_url
 )
+from webhook import WebhookDeliveryService
 
 import psutil, time
 
@@ -107,7 +108,10 @@ async def handle_llm_qa(
             prompt_with_variables=prompt,
             api_token=get_llm_api_key(config),  # Returns None to let litellm handle it
             temperature=get_llm_temperature(config),
-            base_url=get_llm_base_url(config)
+            base_url=get_llm_base_url(config),
+            base_delay=config["llm"].get("backoff_base_delay", 2),
+            max_attempts=config["llm"].get("backoff_max_attempts", 3),
+            exponential_factor=config["llm"].get("backoff_exponential_factor", 2)
         )
 
         return response.choices[0].message.content
@@ -127,10 +131,14 @@ async def process_llm_extraction(
     schema: Optional[str] = None,
     cache: str = "0",
     provider: Optional[str] = None,
+    webhook_config: Optional[Dict] = None,
     temperature: Optional[float] = None,
     base_url: Optional[str] = None
 ) -> None:
     """Process LLM extraction in background."""
+    # Initialize webhook service
+    webhook_service = WebhookDeliveryService(config)
+
     try:
         # Validate provider
         is_valid, error_msg = validate_llm_provider(config, provider)
@@ -139,6 +147,16 @@ async def process_llm_extraction(
                 "status": TaskStatus.FAILED,
                 "error": error_msg
             })
+
+            # Send webhook notification on failure
+            await webhook_service.notify_job_completion(
+                task_id=task_id,
+                task_type="llm_extraction",
+                status="failed",
+                urls=[url],
+                webhook_config=webhook_config,
+                error=error_msg
+            )
             return
         api_key = get_llm_api_key(config, provider)  # Returns None to let litellm handle it
         llm_strategy = LLMExtractionStrategy(
@@ -169,17 +187,40 @@ async def process_llm_extraction(
                 "status": TaskStatus.FAILED,
                 "error": result.error_message
             })
+
+            # Send webhook notification on failure
+            await webhook_service.notify_job_completion(
+                task_id=task_id,
+                task_type="llm_extraction",
+                status="failed",
+                urls=[url],
+                webhook_config=webhook_config,
+                error=result.error_message
+            )
             return
 
         try:
             content = json.loads(result.extracted_content)
         except json.JSONDecodeError:
             content = result.extracted_content
+
+        result_data = {"extracted_content": content}
+
         await redis.hset(f"task:{task_id}", mapping={
             "status": TaskStatus.COMPLETED,
             "result": json.dumps(content)
         })
 
+        # Send webhook notification on successful completion
+        await webhook_service.notify_job_completion(
+            task_id=task_id,
+            task_type="llm_extraction",
+            status="completed",
+            urls=[url],
+            webhook_config=webhook_config,
+            result=result_data
+        )
+
     except Exception as e:
         logger.error(f"LLM extraction error: {str(e)}", exc_info=True)
         await redis.hset(f"task:{task_id}", mapping={
@@ -187,6 +228,16 @@ async def process_llm_extraction(
             "error": str(e)
         })
 
+        # Send webhook notification on failure
+        await webhook_service.notify_job_completion(
+            task_id=task_id,
+            task_type="llm_extraction",
+            status="failed",
+            urls=[url],
+            webhook_config=webhook_config,
+            error=str(e)
+        )
+
 async def handle_markdown_request(
     url: str,
     filter_type: FilterType,
@@ -275,6 +326,7 @@ async def handle_llm_request(
     cache: str = "0",
     config: Optional[dict] = None,
     provider: Optional[str] = None,
+    webhook_config: Optional[Dict] = None,
     temperature: Optional[float] = None,
     api_base_url: Optional[str] = None
 ) -> JSONResponse:
@@ -308,6 +360,7 @@ async def handle_llm_request(
             base_url,
             config,
             provider,
+            webhook_config,
             temperature,
             api_base_url
         )
@@ -355,6 +408,7 @@ async def create_new_task(
     base_url: str,
     config: dict,
     provider: Optional[str] = None,
+    webhook_config: Optional[Dict] = None,
     temperature: Optional[float] = None,
     api_base_url: Optional[str] = None
 ) -> JSONResponse:
@@ -365,12 +419,18 @@ async def create_new_task(
 
     from datetime import datetime
     task_id = f"llm_{int(datetime.now().timestamp())}_{id(background_tasks)}"
-    
-    await redis.hset(f"task:{task_id}", mapping={
+
+    task_data = {
         "status": TaskStatus.PROCESSING,
         "created_at": datetime.now().isoformat(),
         "url": decoded_url
-    })
+    }
+
+    # Store webhook config if provided
+    if webhook_config:
+        task_data["webhook_config"] = json.dumps(webhook_config)
+
+    await redis.hset(f"task:{task_id}", mapping=task_data)
 
     background_tasks.add_task(
         process_llm_extraction,
@@ -382,6 +442,7 @@ async def create_new_task(
         schema,
         cache,
         provider,
+        webhook_config,
         temperature,
         api_base_url
     )
@@ -723,6 +784,7 @@ async def handle_crawl_job(
     browser_config: Dict,
     crawler_config: Dict,
     config: Dict,
+    webhook_config: Optional[Dict] = None,
 ) -> Dict:
     """
     Fire-and-forget version of handle_crawl_request.
@@ -730,13 +792,24 @@ async def handle_crawl_job(
     lets /crawl/job/{task_id} polling fetch the result.
     """
     task_id = f"crawl_{uuid4().hex[:8]}"
-    await redis.hset(f"task:{task_id}", mapping={
+
+    # Store task data in Redis
+    task_data = {
         "status": TaskStatus.PROCESSING,         # <-- keep enum values consistent
         "created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat(),
         "url": json.dumps(urls),                 # store list as JSON string
         "result": "",
         "error": "",
-    })
+    }
+
+    # Store webhook config if provided
+    if webhook_config:
+        task_data["webhook_config"] = json.dumps(webhook_config)
+
+    await redis.hset(f"task:{task_id}", mapping=task_data)
+
+    # Initialize webhook service
+    webhook_service = WebhookDeliveryService(config)
 
     async def _runner():
         try:
@@ -750,6 +823,17 @@ async def handle_crawl_job(
                 "status": TaskStatus.COMPLETED,
                 "result": json.dumps(result),
             })
+
+            # Send webhook notification on successful completion
+            await webhook_service.notify_job_completion(
+                task_id=task_id,
+                task_type="crawl",
+                status="completed",
+                urls=urls,
+                webhook_config=webhook_config,
+                result=result
+            )
+
             await asyncio.sleep(5)  # Give Redis time to process the update
         except Exception as exc:
             await redis.hset(f"task:{task_id}", mapping={
@@ -757,5 +841,15 @@ async def handle_crawl_job(
                 "error": str(exc),
             })
 
+            # Send webhook notification on failure
+            await webhook_service.notify_job_completion(
+                task_id=task_id,
+                task_type="crawl",
+                status="failed",
+                urls=urls,
+                webhook_config=webhook_config,
+                error=str(exc)
+            )
+
     background_tasks.add_task(_runner)
     return {"task_id": task_id}

deploy/docker/config.yml

@@ -37,6 +37,10 @@ rate_limiting:
   storage_uri: "memory://"  # Use "redis://localhost:6379" for production
 
 # Security Configuration
+# WARNING: For production deployments, enable security and use proper SECRET_KEY:
+#   - Set jwt_enabled: true for authentication
+#   - Set SECRET_KEY environment variable to a secure random value
+#   - Set CRAWL4AI_HOOKS_ENABLED=true only if you need hooks (RCE risk)
 security:
   enabled: false
   jwt_enabled: false
@@ -87,4 +91,17 @@ observability:
     enabled: True
     endpoint: "/metrics"
   health_check:
-    endpoint: "/health"
+    endpoint: "/health"
+
+# Webhook Configuration
+webhooks:
+  enabled: true
+  default_url: null  # Optional: default webhook URL for all jobs
+  data_in_payload: false  # Optional: default behavior for including data
+  retry:
+    max_attempts: 5
+    initial_delay_ms: 1000  # 1s, 2s, 4s, 8s, 16s exponential backoff
+    max_delay_ms: 32000
+    timeout_ms: 30000  # 30s timeout per webhook call
+  headers:  # Optional: default headers to include
+    User-Agent: "Crawl4AI-Webhook/1.0"

deploy/docker/hook_manager.py

@@ -117,18 +117,18 @@ class UserHookManager:
         """
         try:
             # Create a safe namespace for the hook
-            # Use a more complete builtins that includes __import__
+            # SECURITY: No __import__ to prevent arbitrary module imports (RCE risk)
             import builtins
             safe_builtins = {}
-            
-            # Add safe built-in functions
+
+            # Add safe built-in functions (no __import__ for security)
             allowed_builtins = [
                 'print', 'len', 'str', 'int', 'float', 'bool',
                 'list', 'dict', 'set', 'tuple', 'range', 'enumerate',
                 'zip', 'map', 'filter', 'any', 'all', 'sum', 'min', 'max',
                 'sorted', 'reversed', 'abs', 'round', 'isinstance', 'type',
                 'getattr', 'hasattr', 'setattr', 'callable', 'iter', 'next',
-                '__import__', '__build_class__'  # Required for exec
+                '__build_class__'  # Required for class definitions in exec
             ]
             
             for name in allowed_builtins:

deploy/docker/job.py

@@ -12,6 +12,7 @@ from api import (
     handle_crawl_job,
     handle_task_status,
 )
+from schemas import WebhookConfig
 
 # ------------- dependency placeholders -------------
 _redis = None        # will be injected from server.py
@@ -37,6 +38,7 @@ class LlmJobPayload(BaseModel):
     schema: Optional[str] = None
     cache:  bool = False
     provider: Optional[str] = None
+    webhook_config: Optional[WebhookConfig] = None
     temperature: Optional[float] = None
     base_url: Optional[str] = None
 
@@ -45,6 +47,7 @@ class CrawlJobPayload(BaseModel):
     urls:           list[HttpUrl]
     browser_config: Dict = {}
     crawler_config: Dict = {}
+    webhook_config: Optional[WebhookConfig] = None
 
 
 # ---------- LL​M job ---------------------------------------------------------
@@ -55,6 +58,10 @@ async def llm_job_enqueue(
         request: Request,
         _td: Dict = Depends(lambda: _token_dep()),   # late-bound dep
 ):
+    webhook_config = None
+    if payload.webhook_config:
+        webhook_config = payload.webhook_config.model_dump(mode='json')
+
     return await handle_llm_request(
         _redis,
         background_tasks,
@@ -65,6 +72,7 @@ async def llm_job_enqueue(
         cache=payload.cache,
         config=_config,
         provider=payload.provider,
+        webhook_config=webhook_config,
         temperature=payload.temperature,
         api_base_url=payload.base_url,
     )
@@ -86,6 +94,10 @@ async def crawl_job_enqueue(
         background_tasks: BackgroundTasks,
         _td: Dict = Depends(lambda: _token_dep()),
 ):
+    webhook_config = None
+    if payload.webhook_config:
+        webhook_config = payload.webhook_config.model_dump(mode='json')
+
     return await handle_crawl_job(
         _redis,
         background_tasks,
@@ -93,6 +105,7 @@ async def crawl_job_enqueue(
         payload.browser_config,
         payload.crawler_config,
         config=_config,
+        webhook_config=webhook_config,
     )
 
 

deploy/docker/requirements.txt

@@ -12,6 +12,6 @@ pydantic>=2.11
 rank-bm25==0.2.2
 anyio==4.9.0
 PyJWT==2.10.1
-mcp>=1.6.0
+mcp>=1.18.0
 websockets>=15.0.1
 httpx[http2]>=0.27.2

deploy/docker/schemas.py

@@ -1,6 +1,6 @@
 from typing import List, Optional, Dict
 from enum import Enum
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, HttpUrl
 from utils import FilterType
 
 
@@ -85,4 +85,22 @@ class JSEndpointRequest(BaseModel):
     scripts: List[str] = Field(
         ...,
         description="List of separated JavaScript snippets to execute"
-    )
+    )
+
+
+class WebhookConfig(BaseModel):
+    """Configuration for webhook notifications."""
+    webhook_url: HttpUrl
+    webhook_data_in_payload: bool = False
+    webhook_headers: Optional[Dict[str, str]] = None
+
+
+class WebhookPayload(BaseModel):
+    """Payload sent to webhook endpoints."""
+    task_id: str
+    task_type: str  # "crawl", "llm_extraction", etc.
+    status: str  # "completed" or "failed"
+    timestamp: str  # ISO 8601 format
+    urls: List[str]
+    error: Optional[str] = None
+    data: Optional[Dict] = None  # Included only if webhook_data_in_payload=True

deploy/docker/server.py

@@ -79,6 +79,10 @@ __version__ = "0.5.1-d1"
 MAX_PAGES = config["crawler"]["pool"].get("max_pages", 30)
 GLOBAL_SEM = asyncio.Semaphore(MAX_PAGES)
 
+# ── security feature flags ───────────────────────────────────
+# Hooks are disabled by default for security (RCE risk). Set to "true" to enable.
+HOOKS_ENABLED = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
+
 # ── default browser config helper ─────────────────────────────
 def get_default_browser_config() -> BrowserConfig:
     """Get default BrowserConfig from config.yml."""
@@ -236,6 +240,19 @@ async def add_security_headers(request: Request, call_next):
         resp.headers.update(config["security"]["headers"])
     return resp
 
+# ───────────────── URL validation helper ─────────────────
+ALLOWED_URL_SCHEMES = ("http://", "https://")
+ALLOWED_URL_SCHEMES_WITH_RAW = ("http://", "https://", "raw:", "raw://")
+
+
+def validate_url_scheme(url: str, allow_raw: bool = False) -> None:
+    """Validate URL scheme to prevent file:// LFI attacks."""
+    allowed = ALLOWED_URL_SCHEMES_WITH_RAW if allow_raw else ALLOWED_URL_SCHEMES
+    if not url.startswith(allowed):
+        schemes = ", ".join(allowed)
+        raise HTTPException(400, f"URL must start with {schemes}")
+
+
 # ───────────────── safe config‑dump helper ─────────────────
 ALLOWED_TYPES = {
     "CrawlerRunConfig": CrawlerRunConfig,
@@ -337,6 +354,7 @@ async def generate_html(
     Crawls the URL, preprocesses the raw HTML for schema extraction, and returns the processed HTML.
     Use when you need sanitized HTML structures for building schemas or further processing.
     """
+    validate_url_scheme(body.url, allow_raw=True)
     from crawler_pool import get_crawler
     cfg = CrawlerRunConfig()
     try:
@@ -368,6 +386,7 @@ async def generate_screenshot(
     Use when you need an image snapshot of the rendered page. Its recommened to provide an output path to save the screenshot.
     Then in result instead of the screenshot you will get a path to the saved file.
     """
+    validate_url_scheme(body.url)
     from crawler_pool import get_crawler
     try:
         cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for)
@@ -402,6 +421,7 @@ async def generate_pdf(
     Use when you need a printable or archivable snapshot of the page. It is recommended to provide an output path to save the PDF.
     Then in result instead of the PDF you will get a path to the saved file.
     """
+    validate_url_scheme(body.url)
     from crawler_pool import get_crawler
     try:
         cfg = CrawlerRunConfig(pdf=True)
@@ -474,6 +494,7 @@ async def execute_js(
         ```
 
     """
+    validate_url_scheme(body.url)
     from crawler_pool import get_crawler
     try:
         cfg = CrawlerRunConfig(js_code=body.scripts)
@@ -600,6 +621,8 @@ async def crawl(
     """
     if not crawl_request.urls:
         raise HTTPException(400, "At least one URL required")
+    if crawl_request.hooks and not HOOKS_ENABLED:
+        raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.")
     # Check whether it is a redirection for a streaming request
     crawler_config = CrawlerRunConfig.load(crawl_request.crawler_config)
     if crawler_config.stream:
@@ -635,6 +658,8 @@ async def crawl_stream(
 ):
     if not crawl_request.urls:
         raise HTTPException(400, "At least one URL required")
+    if crawl_request.hooks and not HOOKS_ENABLED:
+        raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.")
 
     return await stream_process(crawl_request=crawl_request)
 

deploy/docker/tests/run_security_tests.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+"""
+Security Integration Tests for Crawl4AI Docker API.
+Tests that security fixes are working correctly against a running server.
+
+Usage:
+    python run_security_tests.py [base_url]
+
+Example:
+    python run_security_tests.py http://localhost:11235
+"""
+
+import subprocess
+import sys
+import re
+
+# Colors for terminal output
+GREEN = '\033[0;32m'
+RED = '\033[0;31m'
+YELLOW = '\033[1;33m'
+NC = '\033[0m'  # No Color
+
+PASSED = 0
+FAILED = 0
+
+
+def run_curl(args: list) -> str:
+    """Run curl command and return output."""
+    try:
+        result = subprocess.run(
+            ['curl', '-s'] + args,
+            capture_output=True,
+            text=True,
+            timeout=30
+        )
+        return result.stdout + result.stderr
+    except subprocess.TimeoutExpired:
+        return "TIMEOUT"
+    except Exception as e:
+        return str(e)
+
+
+def test_expect(name: str, expect_pattern: str, curl_args: list) -> bool:
+    """Run a test and check if output matches expected pattern."""
+    global PASSED, FAILED
+
+    result = run_curl(curl_args)
+
+    if re.search(expect_pattern, result, re.IGNORECASE):
+        print(f"{GREEN}✓{NC} {name}")
+        PASSED += 1
+        return True
+    else:
+        print(f"{RED}✗{NC} {name}")
+        print(f"  Expected pattern: {expect_pattern}")
+        print(f"  Got: {result[:200]}")
+        FAILED += 1
+        return False
+
+
+def main():
+    global PASSED, FAILED
+
+    base_url = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11235"
+
+    print("=" * 60)
+    print("Crawl4AI Security Integration Tests")
+    print(f"Target: {base_url}")
+    print("=" * 60)
+    print()
+
+    # Check server availability
+    print("Checking server availability...")
+    result = run_curl(['-o', '/dev/null', '-w', '%{http_code}', f'{base_url}/health'])
+    if '200' not in result:
+        print(f"{RED}ERROR: Server not reachable at {base_url}{NC}")
+        print("Please start the server first.")
+        sys.exit(1)
+    print(f"{GREEN}Server is running{NC}")
+    print()
+
+    # === Part A: Security Tests ===
+    print("=== Part A: Security Tests ===")
+    print("(Vulnerabilities must be BLOCKED)")
+    print()
+
+    test_expect(
+        "A1: Hooks disabled by default (403)",
+        r"403|disabled|Hooks are disabled",
+        ['-X', 'POST', f'{base_url}/crawl',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"urls":["https://example.com"],"hooks":{"code":{"on_page_context_created":"async def hook(page, context, **kwargs): return page"}}}']
+    )
+
+    test_expect(
+        "A2: file:// blocked on /execute_js (400)",
+        r"400|must start with",
+        ['-X', 'POST', f'{base_url}/execute_js',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"url":"file:///etc/passwd","scripts":["1"]}']
+    )
+
+    test_expect(
+        "A3: file:// blocked on /screenshot (400)",
+        r"400|must start with",
+        ['-X', 'POST', f'{base_url}/screenshot',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"url":"file:///etc/passwd"}']
+    )
+
+    test_expect(
+        "A4: file:// blocked on /pdf (400)",
+        r"400|must start with",
+        ['-X', 'POST', f'{base_url}/pdf',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"url":"file:///etc/passwd"}']
+    )
+
+    test_expect(
+        "A5: file:// blocked on /html (400)",
+        r"400|must start with",
+        ['-X', 'POST', f'{base_url}/html',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"url":"file:///etc/passwd"}']
+    )
+
+    print()
+
+    # === Part B: Functionality Tests ===
+    print("=== Part B: Functionality Tests ===")
+    print("(Normal operations must WORK)")
+    print()
+
+    test_expect(
+        "B1: Basic crawl works",
+        r"success.*true|results",
+        ['-X', 'POST', f'{base_url}/crawl',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"urls":["https://example.com"]}']
+    )
+
+    test_expect(
+        "B2: /md works with https://",
+        r"success.*true|markdown",
+        ['-X', 'POST', f'{base_url}/md',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"url":"https://example.com"}']
+    )
+
+    test_expect(
+        "B3: Health endpoint works",
+        r"ok",
+        [f'{base_url}/health']
+    )
+
+    print()
+
+    # === Part C: Edge Cases ===
+    print("=== Part C: Edge Cases ===")
+    print("(Malformed input must be REJECTED)")
+    print()
+
+    test_expect(
+        "C1: javascript: URL rejected (400)",
+        r"400|must start with",
+        ['-X', 'POST', f'{base_url}/execute_js',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"url":"javascript:alert(1)","scripts":["1"]}']
+    )
+
+    test_expect(
+        "C2: data: URL rejected (400)",
+        r"400|must start with",
+        ['-X', 'POST', f'{base_url}/execute_js',
+         '-H', 'Content-Type: application/json',
+         '-d', '{"url":"data:text/html,<h1>test</h1>","scripts":["1"]}']
+    )
+
+    print()
+    print("=" * 60)
+    print("Results")
+    print("=" * 60)
+    print(f"Passed: {GREEN}{PASSED}{NC}")
+    print(f"Failed: {RED}{FAILED}{NC}")
+    print()
+
+    if FAILED > 0:
+        print(f"{RED}SOME TESTS FAILED{NC}")
+        sys.exit(1)
+    else:
+        print(f"{GREEN}ALL TESTS PASSED{NC}")
+        sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

deploy/docker/tests/test_security_fixes.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""
+Unit tests for security fixes.
+These tests verify the security fixes at the code level without needing a running server.
+"""
+
+import sys
+import os
+
+# Add parent directory to path to import modules
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import unittest
+
+
+class TestURLValidation(unittest.TestCase):
+    """Test URL scheme validation helper."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        # Import the validation constants and function
+        self.ALLOWED_URL_SCHEMES = ("http://", "https://")
+        self.ALLOWED_URL_SCHEMES_WITH_RAW = ("http://", "https://", "raw:", "raw://")
+
+    def validate_url_scheme(self, url: str, allow_raw: bool = False) -> bool:
+        """Local version of validate_url_scheme for testing."""
+        allowed = self.ALLOWED_URL_SCHEMES_WITH_RAW if allow_raw else self.ALLOWED_URL_SCHEMES
+        return url.startswith(allowed)
+
+    # === SECURITY TESTS: These URLs must be BLOCKED ===
+
+    def test_file_url_blocked(self):
+        """file:// URLs must be blocked (LFI vulnerability)."""
+        self.assertFalse(self.validate_url_scheme("file:///etc/passwd"))
+        self.assertFalse(self.validate_url_scheme("file:///etc/passwd", allow_raw=True))
+
+    def test_file_url_blocked_windows(self):
+        """file:// URLs with Windows paths must be blocked."""
+        self.assertFalse(self.validate_url_scheme("file:///C:/Windows/System32/config/sam"))
+
+    def test_javascript_url_blocked(self):
+        """javascript: URLs must be blocked (XSS)."""
+        self.assertFalse(self.validate_url_scheme("javascript:alert(1)"))
+
+    def test_data_url_blocked(self):
+        """data: URLs must be blocked."""
+        self.assertFalse(self.validate_url_scheme("data:text/html,<script>alert(1)</script>"))
+
+    def test_ftp_url_blocked(self):
+        """ftp: URLs must be blocked."""
+        self.assertFalse(self.validate_url_scheme("ftp://example.com/file"))
+
+    def test_empty_url_blocked(self):
+        """Empty URLs must be blocked."""
+        self.assertFalse(self.validate_url_scheme(""))
+
+    def test_relative_url_blocked(self):
+        """Relative URLs must be blocked."""
+        self.assertFalse(self.validate_url_scheme("/etc/passwd"))
+        self.assertFalse(self.validate_url_scheme("../../../etc/passwd"))
+
+    # === FUNCTIONALITY TESTS: These URLs must be ALLOWED ===
+
+    def test_http_url_allowed(self):
+        """http:// URLs must be allowed."""
+        self.assertTrue(self.validate_url_scheme("http://example.com"))
+        self.assertTrue(self.validate_url_scheme("http://localhost:8080"))
+
+    def test_https_url_allowed(self):
+        """https:// URLs must be allowed."""
+        self.assertTrue(self.validate_url_scheme("https://example.com"))
+        self.assertTrue(self.validate_url_scheme("https://example.com/path?query=1"))
+
+    def test_raw_url_allowed_when_enabled(self):
+        """raw: URLs must be allowed when allow_raw=True."""
+        self.assertTrue(self.validate_url_scheme("raw:<html></html>", allow_raw=True))
+        self.assertTrue(self.validate_url_scheme("raw://<html></html>", allow_raw=True))
+
+    def test_raw_url_blocked_when_disabled(self):
+        """raw: URLs must be blocked when allow_raw=False."""
+        self.assertFalse(self.validate_url_scheme("raw:<html></html>", allow_raw=False))
+        self.assertFalse(self.validate_url_scheme("raw://<html></html>", allow_raw=False))
+
+
+class TestHookBuiltins(unittest.TestCase):
+    """Test that dangerous builtins are removed from hooks."""
+
+    def test_import_not_in_allowed_builtins(self):
+        """__import__ must NOT be in allowed_builtins."""
+        allowed_builtins = [
+            'print', 'len', 'str', 'int', 'float', 'bool',
+            'list', 'dict', 'set', 'tuple', 'range', 'enumerate',
+            'zip', 'map', 'filter', 'any', 'all', 'sum', 'min', 'max',
+            'sorted', 'reversed', 'abs', 'round', 'isinstance', 'type',
+            'getattr', 'hasattr', 'setattr', 'callable', 'iter', 'next',
+            '__build_class__'  # Required for class definitions in exec
+        ]
+
+        self.assertNotIn('__import__', allowed_builtins)
+        self.assertNotIn('eval', allowed_builtins)
+        self.assertNotIn('exec', allowed_builtins)
+        self.assertNotIn('compile', allowed_builtins)
+        self.assertNotIn('open', allowed_builtins)
+
+    def test_build_class_in_allowed_builtins(self):
+        """__build_class__ must be in allowed_builtins (needed for class definitions)."""
+        allowed_builtins = [
+            'print', 'len', 'str', 'int', 'float', 'bool',
+            'list', 'dict', 'set', 'tuple', 'range', 'enumerate',
+            'zip', 'map', 'filter', 'any', 'all', 'sum', 'min', 'max',
+            'sorted', 'reversed', 'abs', 'round', 'isinstance', 'type',
+            'getattr', 'hasattr', 'setattr', 'callable', 'iter', 'next',
+            '__build_class__'
+        ]
+
+        self.assertIn('__build_class__', allowed_builtins)
+
+
+class TestHooksEnabled(unittest.TestCase):
+    """Test HOOKS_ENABLED environment variable logic."""
+
+    def test_hooks_disabled_by_default(self):
+        """Hooks must be disabled by default."""
+        # Simulate the default behavior
+        hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
+
+        # Clear any existing env var to test default
+        original = os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
+        try:
+            hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
+            self.assertFalse(hooks_enabled)
+        finally:
+            if original is not None:
+                os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
+
+    def test_hooks_enabled_when_true(self):
+        """Hooks must be enabled when CRAWL4AI_HOOKS_ENABLED=true."""
+        original = os.environ.get("CRAWL4AI_HOOKS_ENABLED")
+        try:
+            os.environ["CRAWL4AI_HOOKS_ENABLED"] = "true"
+            hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
+            self.assertTrue(hooks_enabled)
+        finally:
+            if original is not None:
+                os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
+            else:
+                os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
+
+    def test_hooks_disabled_when_false(self):
+        """Hooks must be disabled when CRAWL4AI_HOOKS_ENABLED=false."""
+        original = os.environ.get("CRAWL4AI_HOOKS_ENABLED")
+        try:
+            os.environ["CRAWL4AI_HOOKS_ENABLED"] = "false"
+            hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
+            self.assertFalse(hooks_enabled)
+        finally:
+            if original is not None:
+                os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
+            else:
+                os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
+
+
+if __name__ == '__main__':
+    print("=" * 60)
+    print("Crawl4AI Security Fixes - Unit Tests")
+    print("=" * 60)
+    print()
+
+    # Run tests with verbosity
+    unittest.main(verbosity=2)

deploy/docker/webhook.py

@@ -0,0 +1,159 @@
+"""
+Webhook delivery service for Crawl4AI.
+
+This module provides webhook notification functionality with exponential backoff retry logic.
+"""
+import asyncio
+import httpx
+import logging
+from typing import Dict, Optional
+from datetime import datetime, timezone
+
+logger = logging.getLogger(__name__)
+
+
+class WebhookDeliveryService:
+    """Handles webhook delivery with exponential backoff retry logic."""
+
+    def __init__(self, config: Dict):
+        """
+        Initialize the webhook delivery service.
+
+        Args:
+            config: Application configuration dictionary containing webhook settings
+        """
+        self.config = config.get("webhooks", {})
+        self.max_attempts = self.config.get("retry", {}).get("max_attempts", 5)
+        self.initial_delay = self.config.get("retry", {}).get("initial_delay_ms", 1000) / 1000
+        self.max_delay = self.config.get("retry", {}).get("max_delay_ms", 32000) / 1000
+        self.timeout = self.config.get("retry", {}).get("timeout_ms", 30000) / 1000
+
+    async def send_webhook(
+        self,
+        webhook_url: str,
+        payload: Dict,
+        headers: Optional[Dict[str, str]] = None
+    ) -> bool:
+        """
+        Send webhook with exponential backoff retry logic.
+
+        Args:
+            webhook_url: The URL to send the webhook to
+            payload: The JSON payload to send
+            headers: Optional custom headers
+
+        Returns:
+            bool: True if delivered successfully, False otherwise
+        """
+        default_headers = self.config.get("headers", {})
+        merged_headers = {**default_headers, **(headers or {})}
+        merged_headers["Content-Type"] = "application/json"
+
+        async with httpx.AsyncClient(timeout=self.timeout) as client:
+            for attempt in range(self.max_attempts):
+                try:
+                    logger.info(
+                        f"Sending webhook (attempt {attempt + 1}/{self.max_attempts}) to {webhook_url}"
+                    )
+
+                    response = await client.post(
+                        webhook_url,
+                        json=payload,
+                        headers=merged_headers
+                    )
+
+                    # Success or client error (don't retry client errors)
+                    if response.status_code < 500:
+                        if 200 <= response.status_code < 300:
+                            logger.info(f"Webhook delivered successfully to {webhook_url}")
+                            return True
+                        else:
+                            logger.warning(
+                                f"Webhook rejected with status {response.status_code}: {response.text[:200]}"
+                            )
+                            return False  # Client error - don't retry
+
+                    # Server error - retry with backoff
+                    logger.warning(
+                        f"Webhook failed with status {response.status_code}, will retry"
+                    )
+
+                except httpx.TimeoutException as exc:
+                    logger.error(f"Webhook timeout (attempt {attempt + 1}): {exc}")
+                except httpx.RequestError as exc:
+                    logger.error(f"Webhook request error (attempt {attempt + 1}): {exc}")
+                except Exception as exc:
+                    logger.error(f"Webhook delivery error (attempt {attempt + 1}): {exc}")
+
+                # Calculate exponential backoff delay
+                if attempt < self.max_attempts - 1:
+                    delay = min(self.initial_delay * (2 ** attempt), self.max_delay)
+                    logger.info(f"Retrying in {delay}s...")
+                    await asyncio.sleep(delay)
+
+        logger.error(
+            f"Webhook delivery failed after {self.max_attempts} attempts to {webhook_url}"
+        )
+        return False
+
+    async def notify_job_completion(
+        self,
+        task_id: str,
+        task_type: str,
+        status: str,
+        urls: list,
+        webhook_config: Optional[Dict],
+        result: Optional[Dict] = None,
+        error: Optional[str] = None
+    ):
+        """
+        Notify webhook of job completion.
+
+        Args:
+            task_id: The task identifier
+            task_type: Type of task (e.g., "crawl", "llm_extraction")
+            status: Task status ("completed" or "failed")
+            urls: List of URLs that were crawled
+            webhook_config: Webhook configuration from the job request
+            result: Optional crawl result data
+            error: Optional error message if failed
+        """
+        # Determine webhook URL
+        webhook_url = None
+        data_in_payload = self.config.get("data_in_payload", False)
+        custom_headers = None
+
+        if webhook_config:
+            webhook_url = webhook_config.get("webhook_url")
+            data_in_payload = webhook_config.get("webhook_data_in_payload", data_in_payload)
+            custom_headers = webhook_config.get("webhook_headers")
+
+        if not webhook_url:
+            webhook_url = self.config.get("default_url")
+
+        if not webhook_url:
+            logger.debug("No webhook URL configured, skipping notification")
+            return
+
+        # Check if webhooks are enabled
+        if not self.config.get("enabled", True):
+            logger.debug("Webhooks are disabled, skipping notification")
+            return
+
+        # Build payload
+        payload = {
+            "task_id": task_id,
+            "task_type": task_type,
+            "status": status,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "urls": urls
+        }
+
+        if error:
+            payload["error"] = error
+
+        if data_in_payload and result:
+            payload["data"] = result
+
+        # Send webhook (fire and forget - don't block on completion)
+        await self.send_webhook(webhook_url, payload, custom_headers)

docker-compose.yml

@@ -6,15 +6,16 @@ x-base-config: &base-config
     - "11235:11235"  # Gunicorn port
   env_file:
     - .llm.env       # API keys (create from .llm.env.example)
-  environment:
-    - OPENAI_API_KEY=${OPENAI_API_KEY:-}
-    - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
-    - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
-    - GROQ_API_KEY=${GROQ_API_KEY:-}
-    - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
-    - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
-    - GEMINI_API_TOKEN=${GEMINI_API_TOKEN:-}
-    - LLM_PROVIDER=${LLM_PROVIDER:-}  # Optional: Override default provider (e.g., "anthropic/claude-3-opus")
+  # Uncomment to set default environment variables (will overwrite .llm.env)
+  # environment:
+  #   - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+  #   - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
+  #   - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
+  #   - GROQ_API_KEY=${GROQ_API_KEY:-}
+  #   - TOGETHER_API_KEY=${TOGETHER_API_KEY:-}
+  #   - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
+  #   - GEMINI_API_KEY=${GEMINI_API_KEY:-}
+  #   - LLM_PROVIDER=${LLM_PROVIDER:-}  # Optional: Override default provider (e.g., "anthropic/claude-3-opus")
   volumes:
     - /dev/shm:/dev/shm  # Chromium performance
   deploy:

docs/RELEASE_NOTES_v0.8.0.md

@@ -0,0 +1,243 @@
+# Crawl4AI v0.8.0 Release Notes
+
+**Release Date**: January 2026
+**Previous Version**: v0.7.6
+**Status**: Release Candidate
+
+---
+
+## Highlights
+
+- **Critical Security Fixes** for Docker API deployment
+- **11 New Features** including crash recovery, prefetch mode, and proxy improvements
+- **Breaking Changes** - see migration guide below
+
+---
+
+## Breaking Changes
+
+### 1. Docker API: Hooks Disabled by Default
+
+**What changed**: Hooks are now disabled by default on the Docker API.
+
+**Why**: Security fix for Remote Code Execution (RCE) vulnerability.
+
+**Who is affected**: Users of the Docker API who use the `hooks` parameter in `/crawl` requests.
+
+**Migration**:
+```bash
+# To re-enable hooks (only if you trust all API users):
+export CRAWL4AI_HOOKS_ENABLED=true
+```
+
+### 2. Docker API: file:// URLs Blocked
+
+**What changed**: The endpoints `/execute_js`, `/screenshot`, `/pdf`, and `/html` now reject `file://` URLs.
+
+**Why**: Security fix for Local File Inclusion (LFI) vulnerability.
+
+**Who is affected**: Users who were reading local files via the Docker API.
+
+**Migration**: Use the Python library directly for local file processing:
+```python
+# Instead of API call with file:// URL, use library:
+from crawl4ai import AsyncWebCrawler
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(url="file:///path/to/file.html")
+```
+
+---
+
+## Security Fixes
+
+### Critical: Remote Code Execution via Hooks (CVE Pending)
+
+**Severity**: CRITICAL (CVSS 10.0)
+**Affected**: Docker API deployment (all versions before v0.8.0)
+**Vector**: `POST /crawl` with malicious `hooks` parameter
+
+**Details**: The `__import__` builtin was available in hook code, allowing attackers to import `os`, `subprocess`, etc. and execute arbitrary commands.
+
+**Fix**:
+1. Removed `__import__` from allowed builtins
+2. Hooks disabled by default (`CRAWL4AI_HOOKS_ENABLED=false`)
+
+### High: Local File Inclusion via file:// URLs (CVE Pending)
+
+**Severity**: HIGH (CVSS 8.6)
+**Affected**: Docker API deployment (all versions before v0.8.0)
+**Vector**: `POST /execute_js` (and other endpoints) with `file:///etc/passwd`
+
+**Details**: API endpoints accepted `file://` URLs, allowing attackers to read arbitrary files from the server.
+
+**Fix**: URL scheme validation now only allows `http://`, `https://`, and `raw:` URLs.
+
+### Credits
+
+Discovered by **Neo by ProjectDiscovery** ([projectdiscovery.io](https://projectdiscovery.io)) - December 2025
+
+---
+
+## New Features
+
+### 1. init_scripts Support for BrowserConfig
+
+Pre-page-load JavaScript injection for stealth evasions.
+
+```python
+config = BrowserConfig(
+    init_scripts=[
+        "Object.defineProperty(navigator, 'webdriver', {get: () => false})"
+    ]
+)
+```
+
+### 2. CDP Connection Improvements
+
+- WebSocket URL support (`ws://`, `wss://`)
+- Proper cleanup with `cdp_cleanup_on_close=True`
+- Browser reuse across multiple connections
+
+### 3. Crash Recovery for Deep Crawl Strategies
+
+All deep crawl strategies (BFS, DFS, Best-First) now support crash recovery:
+
+```python
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+strategy = BFSDeepCrawlStrategy(
+    max_depth=3,
+    resume_state=saved_state,  # Resume from checkpoint
+    on_state_change=save_callback  # Persist state in real-time
+)
+```
+
+### 4. PDF and MHTML for raw:/file:// URLs
+
+Generate PDFs and MHTML from cached HTML content.
+
+### 5. Screenshots for raw:/file:// URLs
+
+Render cached HTML and capture screenshots.
+
+### 6. base_url Parameter for CrawlerRunConfig
+
+Proper URL resolution for raw: HTML processing:
+
+```python
+config = CrawlerRunConfig(base_url='https://example.com')
+result = await crawler.arun(url='raw:{html}', config=config)
+```
+
+### 7. Prefetch Mode for Two-Phase Deep Crawling
+
+Fast link extraction without full page processing:
+
+```python
+config = CrawlerRunConfig(prefetch=True)
+```
+
+### 8. Proxy Rotation and Configuration
+
+Enhanced proxy rotation with sticky sessions support.
+
+### 9. Proxy Support for HTTP Strategy
+
+Non-browser crawler now supports proxies.
+
+### 10. Browser Pipeline for raw:/file:// URLs
+
+New `process_in_browser` parameter for browser operations on local content:
+
+```python
+config = CrawlerRunConfig(
+    process_in_browser=True,  # Force browser processing
+    screenshot=True
+)
+result = await crawler.arun(url='raw:<html>...</html>', config=config)
+```
+
+### 11. Smart TTL Cache for Sitemap URL Seeder
+
+Intelligent cache invalidation for sitemaps:
+
+```python
+config = SeedingConfig(
+    cache_ttl_hours=24,
+    validate_sitemap_lastmod=True
+)
+```
+
+---
+
+## Bug Fixes
+
+### raw: URL Parsing Truncates at # Character
+
+**Problem**: CSS color codes like `#eee` were being truncated.
+
+**Before**: `raw:body{background:#eee}` → `body{background:`
+**After**: `raw:body{background:#eee}` → `body{background:#eee}`
+
+### Caching System Improvements
+
+Various fixes to cache validation and persistence.
+
+---
+
+## Documentation Updates
+
+- Multi-sample schema generation documentation
+- URL seeder smart TTL cache parameters
+- Security documentation (SECURITY.md)
+
+---
+
+## Upgrade Guide
+
+### From v0.7.x to v0.8.0
+
+1. **Update the package**:
+   ```bash
+   pip install --upgrade crawl4ai
+   ```
+
+2. **Docker API users**:
+   - Hooks are now disabled by default
+   - If you need hooks: `export CRAWL4AI_HOOKS_ENABLED=true`
+   - `file://` URLs no longer work on API (use library directly)
+
+3. **Review security settings**:
+   ```yaml
+   # config.yml - recommended for production
+   security:
+     enabled: true
+     jwt_enabled: true
+   ```
+
+4. **Test your integration** before deploying to production
+
+### Breaking Change Checklist
+
+- [ ] Check if you use `hooks` parameter in API calls
+- [ ] Check if you use `file://` URLs via the API
+- [ ] Update environment variables if needed
+- [ ] Review security configuration
+
+---
+
+## Full Changelog
+
+See [CHANGELOG.md](../CHANGELOG.md) for complete version history.
+
+---
+
+## Contributors
+
+Thanks to all contributors who made this release possible.
+
+Special thanks to **Neo by ProjectDiscovery** for responsible security disclosure.
+
+---
+
+*For questions or issues, please open a [GitHub Issue](https://github.com/unclecode/crawl4ai/issues).*

docs/blog/release-v0.7.4.md

@@ -10,7 +10,6 @@ Today I'm releasing Crawl4AI v0.7.4—the Intelligent Table Extraction & Perform
 
 - **🚀 LLMTableExtraction**: Revolutionary table extraction with intelligent chunking for massive tables
 - **⚡ Enhanced Concurrency**: True concurrency improvements for fast-completing tasks in batch operations
-- **🧹 Memory Management Refactor**: Streamlined memory utilities and better resource management
 - **🔧 Browser Manager Fixes**: Resolved race conditions in concurrent page creation
 - **⌨️ Cross-Platform Browser Profiler**: Improved keyboard handling and quit mechanisms
 - **🔗 Advanced URL Processing**: Better handling of raw URLs and base tag link resolution
@@ -158,40 +157,6 @@ async with AsyncWebCrawler() as crawler:
 - **Monitoring Systems**: Faster health checks and status page monitoring
 - **Data Aggregation**: Improved performance for real-time data collection
 
-## 🧹 Memory Management Refactor: Cleaner Architecture
-
-**The Problem:** Memory utilities were scattered and difficult to maintain, with potential import conflicts and unclear organization.
-
-**My Solution:** I consolidated all memory-related utilities into the main `utils.py` module, creating a cleaner, more maintainable architecture.
-
-### Improved Memory Handling
-
-```python
-# All memory utilities now consolidated
-from crawl4ai.utils import get_true_memory_usage_percent, MemoryMonitor
-
-# Enhanced memory monitoring
-monitor = MemoryMonitor()
-monitor.start_monitoring()
-
-async with AsyncWebCrawler() as crawler:
-    # Memory-efficient batch processing
-    results = await crawler.arun_many(large_url_list)
-    
-    # Get accurate memory metrics
-    memory_usage = get_true_memory_usage_percent()
-    memory_report = monitor.get_report()
-    
-    print(f"Memory efficiency: {memory_report['efficiency']:.1f}%")
-    print(f"Peak usage: {memory_report['peak_mb']:.1f} MB")
-```
-
-**Expected Real-World Impact:**
-- **Production Stability**: More reliable memory tracking and management
-- **Code Maintainability**: Cleaner architecture for easier debugging
-- **Import Clarity**: Resolved potential conflicts and import issues
-- **Developer Experience**: Simpler API for memory monitoring
-
 ## 🔧 Critical Stability Fixes
 
 ### Browser Manager Race Condition Resolution

docs/blog/release-v0.7.5.md

@@ -0,0 +1,318 @@
+# 🚀 Crawl4AI v0.7.5: The Docker Hooks & Security Update
+
+*September 29, 2025 • 8 min read*
+
+---
+
+Today I'm releasing Crawl4AI v0.7.5—focused on extensibility and security. This update introduces the Docker Hooks System for pipeline customization, enhanced LLM integration, and important security improvements.
+
+## 🎯 What's New at a Glance
+
+- **Docker Hooks System**: Custom Python functions at key pipeline points with function-based API
+- **Function-Based Hooks**: New `hooks_to_string()` utility with Docker client auto-conversion
+- **Enhanced LLM Integration**: Custom providers with temperature control
+- **HTTPS Preservation**: Secure internal link handling
+- **Bug Fixes**: Resolved multiple community-reported issues
+- **Improved Docker Error Handling**: Better debugging and reliability
+
+## 🔧 Docker Hooks System: Pipeline Customization
+
+Every scraping project needs custom logic—authentication, performance optimization, content processing. Traditional solutions require forking or complex workarounds. Docker Hooks let you inject custom Python functions at 8 key points in the crawling pipeline.
+
+### Real Example: Authentication & Performance
+
+```python
+import requests
+
+# Real working hooks for httpbin.org
+hooks_config = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("Hook: Setting up page context")
+    # Block images to speed up crawling
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    print("Hook: Images blocked")
+    return page
+""",
+
+    "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    print("Hook: Before retrieving HTML")
+    # Scroll to bottom to load lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    print("Hook: Scrolled to bottom")
+    return page
+""",
+
+    "before_goto": """
+async def hook(page, context, url, **kwargs):
+    print(f"Hook: About to navigate to {url}")
+    # Add custom headers
+    await page.set_extra_http_headers({
+        'X-Test-Header': 'crawl4ai-hooks-test'
+    })
+    return page
+"""
+}
+
+# Test with Docker API
+payload = {
+    "urls": ["https://httpbin.org/html"],
+    "hooks": {
+        "code": hooks_config,
+        "timeout": 30
+    }
+}
+
+response = requests.post("http://localhost:11235/crawl", json=payload)
+result = response.json()
+
+if result.get('success'):
+    print("✅ Hooks executed successfully!")
+    print(f"Content length: {len(result.get('markdown', ''))} characters")
+```
+
+**Available Hook Points:**
+- `on_browser_created`: Browser setup
+- `on_page_context_created`: Page context configuration
+- `before_goto`: Pre-navigation setup
+- `after_goto`: Post-navigation processing
+- `on_user_agent_updated`: User agent changes
+- `on_execution_started`: Crawl initialization
+- `before_retrieve_html`: Pre-extraction processing
+- `before_return_html`: Final HTML processing
+
+### Function-Based Hooks API
+
+Writing hooks as strings works, but lacks IDE support and type checking. v0.7.5 introduces a function-based approach with automatic conversion!
+
+**Option 1: Using the `hooks_to_string()` Utility**
+
+```python
+from crawl4ai import hooks_to_string
+import requests
+
+# Define hooks as regular Python functions (with full IDE support!)
+async def on_page_context_created(page, context, **kwargs):
+    """Block images to speed up crawling"""
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+
+async def before_goto(page, context, url, **kwargs):
+    """Add custom headers"""
+    await page.set_extra_http_headers({
+        'X-Crawl4AI': 'v0.7.5',
+        'X-Custom-Header': 'my-value'
+    })
+    return page
+
+# Convert functions to strings
+hooks_code = hooks_to_string({
+    "on_page_context_created": on_page_context_created,
+    "before_goto": before_goto
+})
+
+# Use with REST API
+payload = {
+    "urls": ["https://httpbin.org/html"],
+    "hooks": {"code": hooks_code, "timeout": 30}
+}
+response = requests.post("http://localhost:11235/crawl", json=payload)
+```
+
+**Option 2: Docker Client with Automatic Conversion (Recommended!)**
+
+```python
+from crawl4ai.docker_client import Crawl4aiDockerClient
+
+# Define hooks as functions (same as above)
+async def on_page_context_created(page, context, **kwargs):
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    return page
+
+async def before_retrieve_html(page, context, **kwargs):
+    # Scroll to load lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    return page
+
+# Use Docker client - conversion happens automatically!
+client = Crawl4aiDockerClient(base_url="http://localhost:11235")
+
+results = await client.crawl(
+    urls=["https://httpbin.org/html"],
+    hooks={
+        "on_page_context_created": on_page_context_created,
+        "before_retrieve_html": before_retrieve_html
+    },
+    hooks_timeout=30
+)
+
+if results and results.success:
+    print(f"✅ Hooks executed! HTML length: {len(results.html)}")
+```
+
+**Benefits of Function-Based Hooks:**
+- ✅ Full IDE support (autocomplete, syntax highlighting)
+- ✅ Type checking and linting
+- ✅ Easier to test and debug
+- ✅ Reusable across projects
+- ✅ Automatic conversion in Docker client
+- ✅ No breaking changes - string hooks still work!
+
+## 🤖 Enhanced LLM Integration
+
+Enhanced LLM integration with custom providers, temperature control, and base URL configuration.
+
+### Multi-Provider Support
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+# Test with different providers
+async def test_llm_providers():
+    # OpenAI with custom temperature
+    openai_strategy = LLMExtractionStrategy(
+        provider="gemini/gemini-2.5-flash-lite",
+        api_token="your-api-token",
+        temperature=0.7,  # New in v0.7.5
+        instruction="Summarize this page in one sentence"
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            config=CrawlerRunConfig(extraction_strategy=openai_strategy)
+        )
+
+        if result.success:
+            print("✅ LLM extraction completed")
+            print(result.extracted_content)
+
+# Docker API with enhanced LLM config
+llm_payload = {
+    "url": "https://example.com",
+    "f": "llm",
+    "q": "Summarize this page in one sentence.",
+    "provider": "gemini/gemini-2.5-flash-lite",
+    "temperature": 0.7
+}
+
+response = requests.post("http://localhost:11235/md", json=llm_payload)
+```
+
+**New Features:**
+- Custom `temperature` parameter for creativity control
+- `base_url` for custom API endpoints
+- Multi-provider environment variable support
+- Docker API integration
+
+## 🔒 HTTPS Preservation
+
+**The Problem:** Modern web apps require HTTPS everywhere. When crawlers downgrade internal links from HTTPS to HTTP, authentication breaks and security warnings appear.
+
+**Solution:** HTTPS preservation maintains secure protocols throughout crawling.
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, FilterChain, URLPatternFilter, BFSDeepCrawlStrategy
+
+async def test_https_preservation():
+    # Enable HTTPS preservation
+    url_filter = URLPatternFilter(
+        patterns=["^(https:\/\/)?quotes\.toscrape\.com(\/.*)?$"]
+    )
+
+    config = CrawlerRunConfig(
+        exclude_external_links=True,
+        preserve_https_for_internal_links=True,  # New in v0.7.5
+        deep_crawl_strategy=BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=5,
+            filter_chain=FilterChain([url_filter])
+        )
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        async for result in await crawler.arun(
+            url="https://quotes.toscrape.com",
+            config=config
+        ):
+            # All internal links maintain HTTPS
+            internal_links = [link['href'] for link in result.links['internal']]
+            https_links = [link for link in internal_links if link.startswith('https://')]
+
+            print(f"HTTPS links preserved: {len(https_links)}/{len(internal_links)}")
+            for link in https_links[:3]:
+                print(f"  → {link}")
+```
+
+## 🛠️ Bug Fixes and Improvements
+
+### Major Fixes
+- **URL Processing**: Fixed '+' sign preservation in query parameters (#1332)
+- **Proxy Configuration**: Enhanced proxy string parsing (old `proxy` parameter deprecated)
+- **Docker Error Handling**: Comprehensive error messages with status codes
+- **Memory Management**: Fixed leaks in long-running sessions
+- **JWT Authentication**: Fixed Docker JWT validation issues (#1442)
+- **Playwright Stealth**: Fixed stealth features for Playwright integration (#1481)
+- **API Configuration**: Fixed config handling to prevent overriding user-provided settings (#1505)
+- **Docker Filter Serialization**: Resolved JSON encoding errors in deep crawl strategy (#1419)
+- **LLM Provider Support**: Fixed custom LLM provider integration for adaptive crawler (#1291)
+- **Performance Issues**: Resolved backoff strategy failures and timeout handling (#989)
+
+### Community-Reported Issues Fixed
+This release addresses multiple issues reported by the community through GitHub issues and Discord discussions:
+- Fixed browser configuration reference errors
+- Resolved dependency conflicts with cssselect
+- Improved error messaging for failed authentications
+- Enhanced compatibility with various proxy configurations
+- Fixed edge cases in URL normalization
+
+### Configuration Updates
+```python
+# Old proxy config (deprecated)
+# browser_config = BrowserConfig(proxy="http://proxy:8080")
+
+# New enhanced proxy config
+browser_config = BrowserConfig(
+    proxy_config={
+        "server": "http://proxy:8080",
+        "username": "optional-user",
+        "password": "optional-pass"
+    }
+)
+```
+
+## 🔄 Breaking Changes
+
+1. **Python 3.10+ Required**: Upgrade from Python 3.9
+2. **Proxy Parameter Deprecated**: Use new `proxy_config` structure
+3. **New Dependency**: Added `cssselect` for better CSS handling
+
+## 🚀 Get Started
+
+```bash
+# Install latest version
+pip install crawl4ai==0.7.5
+
+# Docker deployment
+docker pull unclecode/crawl4ai:latest
+docker run -p 11235:11235 unclecode/crawl4ai:latest
+```
+
+**Try the Demo:**
+```bash
+# Run working examples
+python docs/releases_review/demo_v0.7.5.py
+```
+
+**Resources:**
+- 📖 Documentation: [docs.crawl4ai.com](https://docs.crawl4ai.com)
+- 🐙 GitHub: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
+- 💬 Discord: [discord.gg/crawl4ai](https://discord.gg/jP8KfhDhyN)
+- 🐦 Twitter: [@unclecode](https://x.com/unclecode)
+
+Happy crawling! 🕷️

docs/blog/release-v0.7.6.md

@@ -0,0 +1,314 @@
+# Crawl4AI v0.7.6 Release Notes
+
+*Release Date: October 22, 2025*
+
+I'm excited to announce Crawl4AI v0.7.6, featuring a complete webhook infrastructure for the Docker job queue API! This release eliminates polling and brings real-time notifications to both crawling and LLM extraction workflows.
+
+## 🎯 What's New
+
+### Webhook Support for Docker Job Queue API
+
+The headline feature of v0.7.6 is comprehensive webhook support for asynchronous job processing. No more constant polling to check if your jobs are done - get instant notifications when they complete!
+
+**Key Capabilities:**
+
+- ✅ **Universal Webhook Support**: Both `/crawl/job` and `/llm/job` endpoints now support webhooks
+- ✅ **Flexible Delivery Modes**: Choose notification-only or include full data in the webhook payload
+- ✅ **Reliable Delivery**: Exponential backoff retry mechanism (5 attempts: 1s → 2s → 4s → 8s → 16s)
+- ✅ **Custom Authentication**: Add custom headers for webhook authentication
+- ✅ **Global Configuration**: Set default webhook URL in `config.yml` for all jobs
+- ✅ **Task Type Identification**: Distinguish between `crawl` and `llm_extraction` tasks
+
+### How It Works
+
+Instead of constantly checking job status:
+
+**OLD WAY (Polling):**
+```python
+# Submit job
+response = requests.post("http://localhost:11235/crawl/job", json=payload)
+task_id = response.json()['task_id']
+
+# Poll until complete
+while True:
+    status = requests.get(f"http://localhost:11235/crawl/job/{task_id}")
+    if status.json()['status'] == 'completed':
+        break
+    time.sleep(5)  # Wait and try again
+```
+
+**NEW WAY (Webhooks):**
+```python
+# Submit job with webhook
+payload = {
+    "urls": ["https://example.com"],
+    "webhook_config": {
+        "webhook_url": "https://myapp.com/webhook",
+        "webhook_data_in_payload": True
+    }
+}
+response = requests.post("http://localhost:11235/crawl/job", json=payload)
+
+# Done! Webhook will notify you when complete
+# Your webhook handler receives the results automatically
+```
+
+### Crawl Job Webhooks
+
+```bash
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"],
+    "browser_config": {"headless": true},
+    "crawler_config": {"cache_mode": "bypass"},
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
+      "webhook_data_in_payload": false,
+      "webhook_headers": {
+        "X-Webhook-Secret": "your-secret-token"
+      }
+    }
+  }'
+```
+
+### LLM Extraction Job Webhooks (NEW!)
+
+```bash
+curl -X POST http://localhost:11235/llm/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "url": "https://example.com/article",
+    "q": "Extract the article title, author, and publication date",
+    "schema": "{\"type\":\"object\",\"properties\":{\"title\":{\"type\":\"string\"}}}",
+    "provider": "openai/gpt-4o-mini",
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/llm-complete",
+      "webhook_data_in_payload": true
+    }
+  }'
+```
+
+### Webhook Payload Structure
+
+**Success (with data):**
+```json
+{
+  "task_id": "llm_1698765432",
+  "task_type": "llm_extraction",
+  "status": "completed",
+  "timestamp": "2025-10-22T10:30:00.000000+00:00",
+  "urls": ["https://example.com/article"],
+  "data": {
+    "extracted_content": {
+      "title": "Understanding Web Scraping",
+      "author": "John Doe",
+      "date": "2025-10-22"
+    }
+  }
+}
+```
+
+**Failure:**
+```json
+{
+  "task_id": "crawl_abc123",
+  "task_type": "crawl",
+  "status": "failed",
+  "timestamp": "2025-10-22T10:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "error": "Connection timeout after 30s"
+}
+```
+
+### Simple Webhook Handler Example
+
+```python
+from flask import Flask, request, jsonify
+
+app = Flask(__name__)
+
+@app.route('/webhook', methods=['POST'])
+def handle_webhook():
+    payload = request.json
+
+    task_id = payload['task_id']
+    task_type = payload['task_type']
+    status = payload['status']
+
+    if status == 'completed':
+        if 'data' in payload:
+            # Process data directly
+            data = payload['data']
+        else:
+            # Fetch from API
+            endpoint = 'crawl' if task_type == 'crawl' else 'llm'
+            response = requests.get(f'http://localhost:11235/{endpoint}/job/{task_id}')
+            data = response.json()
+
+        # Your business logic here
+        print(f"Job {task_id} completed!")
+
+    elif status == 'failed':
+        error = payload.get('error', 'Unknown error')
+        print(f"Job {task_id} failed: {error}")
+
+    return jsonify({"status": "received"}), 200
+
+app.run(port=8080)
+```
+
+## 📊 Performance Improvements
+
+- **Reduced Server Load**: Eliminates constant polling requests
+- **Lower Latency**: Instant notification vs. polling interval delay
+- **Better Resource Usage**: Frees up client connections while jobs run in background
+- **Scalable Architecture**: Handles high-volume crawling workflows efficiently
+
+## 🐛 Bug Fixes
+
+- Fixed webhook configuration serialization for Pydantic HttpUrl fields
+- Improved error handling in webhook delivery service
+- Enhanced Redis task storage for webhook config persistence
+
+## 🌍 Expected Real-World Impact
+
+### For Web Scraping Workflows
+- **Reduced Costs**: Less API calls = lower bandwidth and server costs
+- **Better UX**: Instant notifications improve user experience
+- **Scalability**: Handle 100s of concurrent jobs without polling overhead
+
+### For LLM Extraction Pipelines
+- **Async Processing**: Submit LLM extraction jobs and move on
+- **Batch Processing**: Queue multiple extractions, get notified as they complete
+- **Integration**: Easy integration with workflow automation tools (Zapier, n8n, etc.)
+
+### For Microservices
+- **Event-Driven**: Perfect for event-driven microservice architectures
+- **Decoupling**: Decouple job submission from result processing
+- **Reliability**: Automatic retries ensure webhooks are delivered
+
+## 🔄 Breaking Changes
+
+**None!** This release is fully backward compatible.
+
+- Webhook configuration is optional
+- Existing code continues to work without modification
+- Polling is still supported for jobs without webhook config
+
+## 📚 Documentation
+
+### New Documentation
+- **[WEBHOOK_EXAMPLES.md](../deploy/docker/WEBHOOK_EXAMPLES.md)** - Comprehensive webhook usage guide
+- **[docker_webhook_example.py](../docs/examples/docker_webhook_example.py)** - Working code examples
+
+### Updated Documentation
+- **[Docker README](../deploy/docker/README.md)** - Added webhook sections
+- API documentation with webhook examples
+
+## 🛠️ Migration Guide
+
+No migration needed! Webhooks are opt-in:
+
+1. **To use webhooks**: Add `webhook_config` to your job payload
+2. **To keep polling**: Continue using your existing code
+
+### Quick Start
+
+```python
+# Just add webhook_config to your existing payload
+payload = {
+    # Your existing configuration
+    "urls": ["https://example.com"],
+    "browser_config": {...},
+    "crawler_config": {...},
+
+    # NEW: Add webhook configuration
+    "webhook_config": {
+        "webhook_url": "https://myapp.com/webhook",
+        "webhook_data_in_payload": True
+    }
+}
+```
+
+## 🔧 Configuration
+
+### Global Webhook Configuration (config.yml)
+
+```yaml
+webhooks:
+  enabled: true
+  default_url: "https://myapp.com/webhooks/default"  # Optional
+  data_in_payload: false
+  retry:
+    max_attempts: 5
+    initial_delay_ms: 1000
+    max_delay_ms: 32000
+    timeout_ms: 30000
+  headers:
+    User-Agent: "Crawl4AI-Webhook/1.0"
+```
+
+## 🚀 Upgrade Instructions
+
+### Docker
+
+```bash
+# Pull the latest image
+docker pull unclecode/crawl4ai:0.7.6
+
+# Or use latest tag
+docker pull unclecode/crawl4ai:latest
+
+# Run with webhook support
+docker run -d \
+  -p 11235:11235 \
+  --env-file .llm.env \
+  --name crawl4ai \
+  unclecode/crawl4ai:0.7.6
+```
+
+### Python Package
+
+```bash
+pip install --upgrade crawl4ai
+```
+
+## 💡 Pro Tips
+
+1. **Use notification-only mode** for large results - fetch data separately to avoid large webhook payloads
+2. **Set custom headers** for webhook authentication and request tracking
+3. **Configure global default webhook** for consistent handling across all jobs
+4. **Implement idempotent webhook handlers** - same webhook may be delivered multiple times on retry
+5. **Use structured schemas** with LLM extraction for predictable webhook data
+
+## 🎬 Demo
+
+Try the release demo:
+
+```bash
+python docs/releases_review/demo_v0.7.6.py
+```
+
+This comprehensive demo showcases:
+- Crawl job webhooks (notification-only and with data)
+- LLM extraction webhooks (with JSON schema support)
+- Custom headers for authentication
+- Webhook retry mechanism
+- Real-time webhook receiver
+
+## 🙏 Acknowledgments
+
+Thank you to the community for the feedback that shaped this feature! Special thanks to everyone who requested webhook support for asynchronous job processing.
+
+## 📞 Support
+
+- **Documentation**: https://docs.crawl4ai.com
+- **GitHub Issues**: https://github.com/unclecode/crawl4ai/issues
+- **Discord**: https://discord.gg/crawl4ai
+
+---
+
+**Happy crawling with webhooks!** 🕷️🪝
+
+*- unclecode*

docs/blog/release-v0.7.7.md

@@ -0,0 +1,626 @@
+# 🚀 Crawl4AI v0.7.7: The Self-Hosting & Monitoring Update
+
+*November 14, 2025 • 10 min read*
+
+---
+
+Today I'm releasing Crawl4AI v0.7.7—the Self-Hosting & Monitoring Update. This release transforms Crawl4AI Docker from a simple containerized crawler into a complete self-hosting platform with enterprise-grade real-time monitoring, full operational transparency, and production-ready observability.
+
+## 🎯 What's New at a Glance
+
+- **📊 Real-time Monitoring Dashboard**: Interactive web UI with live system metrics and browser pool status
+- **🔌 Comprehensive Monitor API**: Complete REST API for programmatic access to all monitoring data
+- **⚡ WebSocket Streaming**: Real-time updates every 2 seconds for custom dashboards
+- **🎮 Control Actions**: Manual browser management (kill, restart, cleanup)
+- **🔥 Smart Browser Pool**: 3-tier architecture (permanent/hot/cold) with automatic promotion
+- **🧹 Janitor Cleanup System**: Automatic resource management with event logging
+- **📈 Production Metrics**: 6 critical metrics for operational excellence
+- **🏭 Integration Ready**: Prometheus, alerting, and log aggregation examples
+- **🐛 Critical Bug Fixes**: Async LLM extraction, DFS crawling, viewport config, and more
+
+## 📊 Real-time Monitoring Dashboard: Complete Visibility
+
+**The Problem:** Running Crawl4AI in Docker was like flying blind. Users had no visibility into what was happening inside the container—memory usage, active requests, browser pools, or errors. Troubleshooting required checking logs, and there was no way to monitor performance or manually intervene when issues occurred.
+
+**My Solution:** I built a complete real-time monitoring system with an interactive dashboard, comprehensive REST API, WebSocket streaming, and manual control actions. Now you have full transparency and control over your crawling infrastructure.
+
+### The Self-Hosting Value Proposition
+
+Before v0.7.7, Docker was just a containerized crawler. After v0.7.7, it's a complete self-hosting platform that gives you:
+
+- **🔒 Data Privacy**: Your data never leaves your infrastructure
+- **💰 Cost Control**: No per-request pricing or rate limits
+- **🎯 Full Customization**: Complete control over configurations and strategies
+- **📊 Complete Transparency**: Real-time visibility into every aspect
+- **⚡ Performance**: Direct access without network overhead
+- **🛡️ Enterprise Security**: Keep workflows behind your firewall
+
+### Interactive Monitoring Dashboard
+
+Access the dashboard at `http://localhost:11235/dashboard` to see:
+
+- **System Health Overview**: CPU, memory, network, and uptime in real-time
+- **Live Request Tracking**: Active and completed requests with full details
+- **Browser Pool Management**: Interactive table with permanent/hot/cold browsers
+- **Janitor Events Log**: Automatic cleanup activities
+- **Error Monitoring**: Full context error logs
+
+The dashboard updates every 2 seconds via WebSocket, giving you live visibility into your crawling operations.
+
+## 🔌 Monitor API: Programmatic Access
+
+**The Problem:** Monitoring dashboards are great for humans, but automation and integration require programmatic access.
+
+**My Solution:** A comprehensive REST API that exposes all monitoring data for integration with your existing infrastructure.
+
+### System Health Endpoint
+
+```python
+import httpx
+import asyncio
+
+async def monitor_system_health():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/health")
+        health = response.json()
+
+        print(f"Container Metrics:")
+        print(f"  CPU: {health['container']['cpu_percent']:.1f}%")
+        print(f"  Memory: {health['container']['memory_percent']:.1f}%")
+        print(f"  Uptime: {health['container']['uptime_seconds']}s")
+
+        print(f"\nBrowser Pool:")
+        print(f"  Permanent: {health['pool']['permanent']['active']} active")
+        print(f"  Hot Pool: {health['pool']['hot']['count']} browsers")
+        print(f"  Cold Pool: {health['pool']['cold']['count']} browsers")
+
+        print(f"\nStatistics:")
+        print(f"  Total Requests: {health['stats']['total_requests']}")
+        print(f"  Success Rate: {health['stats']['success_rate_percent']:.1f}%")
+        print(f"  Avg Latency: {health['stats']['avg_latency_ms']:.0f}ms")
+
+asyncio.run(monitor_system_health())
+```
+
+### Request Tracking
+
+```python
+async def track_requests():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/requests")
+        requests_data = response.json()
+
+        print(f"Active Requests: {len(requests_data['active'])}")
+        print(f"Completed Requests: {len(requests_data['completed'])}")
+
+        # See details of recent requests
+        for req in requests_data['completed'][:5]:
+            status_icon = "✅" if req['success'] else "❌"
+            print(f"{status_icon} {req['endpoint']} - {req['latency_ms']:.0f}ms")
+```
+
+### Browser Pool Management
+
+```python
+async def monitor_browser_pool():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/browsers")
+        browsers = response.json()
+
+        print(f"Pool Summary:")
+        print(f"  Total Browsers: {browsers['summary']['total_count']}")
+        print(f"  Total Memory: {browsers['summary']['total_memory_mb']} MB")
+        print(f"  Reuse Rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
+
+        # List all browsers
+        for browser in browsers['permanent']:
+            print(f"🔥 Permanent: {browser['browser_id'][:8]}... | "
+                  f"Requests: {browser['request_count']} | "
+                  f"Memory: {browser['memory_mb']:.0f} MB")
+```
+
+### Endpoint Performance Statistics
+
+```python
+async def get_endpoint_stats():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/endpoints/stats")
+        stats = response.json()
+
+        print("Endpoint Analytics:")
+        for endpoint, data in stats.items():
+            print(f"  {endpoint}:")
+            print(f"    Requests: {data['count']}")
+            print(f"    Avg Latency: {data['avg_latency_ms']:.0f}ms")
+            print(f"    Success Rate: {data['success_rate_percent']:.1f}%")
+```
+
+### Complete API Reference
+
+The Monitor API includes these endpoints:
+
+- `GET /monitor/health` - System health with pool statistics
+- `GET /monitor/requests` - Active and completed request tracking
+- `GET /monitor/browsers` - Browser pool details and efficiency
+- `GET /monitor/endpoints/stats` - Per-endpoint performance analytics
+- `GET /monitor/timeline?minutes=5` - Time-series data for charts
+- `GET /monitor/logs/janitor?limit=10` - Cleanup activity logs
+- `GET /monitor/logs/errors?limit=10` - Error logs with context
+- `POST /monitor/actions/cleanup` - Force immediate cleanup
+- `POST /monitor/actions/kill_browser` - Kill specific browser
+- `POST /monitor/actions/restart_browser` - Restart browser
+- `POST /monitor/stats/reset` - Reset accumulated statistics
+
+## ⚡ WebSocket Streaming: Real-time Updates
+
+**The Problem:** Polling the API every few seconds wastes resources and adds latency. Real-time dashboards need instant updates.
+
+**My Solution:** WebSocket streaming with 2-second update intervals for building custom real-time dashboards.
+
+### WebSocket Integration Example
+
+```python
+import websockets
+import json
+import asyncio
+
+async def monitor_realtime():
+    uri = "ws://localhost:11235/monitor/ws"
+
+    async with websockets.connect(uri) as websocket:
+        print("Connected to real-time monitoring stream")
+
+        while True:
+            # Receive update every 2 seconds
+            data = await websocket.recv()
+            update = json.loads(data)
+
+            # Access all monitoring data
+            print(f"\n--- Update at {update['timestamp']} ---")
+            print(f"Memory: {update['health']['container']['memory_percent']:.1f}%")
+            print(f"Active Requests: {len(update['requests']['active'])}")
+            print(f"Total Browsers: {update['browsers']['summary']['total_count']}")
+
+            if update['errors']:
+                print(f"⚠️  Recent Errors: {len(update['errors'])}")
+
+asyncio.run(monitor_realtime())
+```
+
+**Expected Real-World Impact:**
+- **Custom Dashboards**: Build tailored monitoring UIs for your team
+- **Real-time Alerting**: Trigger alerts instantly when metrics exceed thresholds
+- **Integration**: Feed live data into monitoring tools like Grafana
+- **Automation**: React to events in real-time without polling
+
+## 🔥 Smart Browser Pool: 3-Tier Architecture
+
+**The Problem:** Creating a new browser for every request is slow and memory-intensive. Traditional browser pools are static and inefficient.
+
+**My Solution:** A smart 3-tier browser pool that automatically adapts to usage patterns.
+
+### How It Works
+
+```python
+import httpx
+
+async def demonstrate_browser_pool():
+    async with httpx.AsyncClient() as client:
+        # Request 1-3: Default config → Uses permanent browser
+        print("Phase 1: Using permanent browser")
+        for i in range(3):
+            await client.post(
+                "http://localhost:11235/crawl",
+                json={"urls": [f"https://httpbin.org/html?req={i}"]}
+            )
+            print(f"  Request {i+1}: Reused permanent browser")
+
+        # Request 4-6: Custom viewport → Cold pool (first use)
+        print("\nPhase 2: Custom config creates cold pool browser")
+        viewport_config = {"viewport": {"width": 1280, "height": 720}}
+        for i in range(4):
+            await client.post(
+                "http://localhost:11235/crawl",
+                json={
+                    "urls": [f"https://httpbin.org/json?v={i}"],
+                    "browser_config": viewport_config
+                }
+            )
+            if i < 2:
+                print(f"  Request {i+1}: Cold pool browser")
+            else:
+                print(f"  Request {i+1}: Promoted to hot pool! (after 3 uses)")
+
+        # Check pool status
+        response = await client.get("http://localhost:11235/monitor/browsers")
+        browsers = response.json()
+
+        print(f"\nPool Status:")
+        print(f"  Permanent: {len(browsers['permanent'])} (always active)")
+        print(f"  Hot: {len(browsers['hot'])} (frequently used configs)")
+        print(f"  Cold: {len(browsers['cold'])} (on-demand)")
+        print(f"  Reuse Rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
+
+asyncio.run(demonstrate_browser_pool())
+```
+
+**Pool Tiers:**
+
+- **🔥 Permanent Browser**: Always-on, default configuration, instant response
+- **♨️ Hot Pool**: Browsers promoted after 3+ uses, kept warm for quick access
+- **❄️ Cold Pool**: On-demand browsers for variant configs, cleaned up when idle
+
+**Expected Real-World Impact:**
+- **Memory Efficiency**: 10x reduction in memory usage vs creating browsers per request
+- **Performance**: Instant access to frequently-used configurations
+- **Automatic Optimization**: Pool adapts to your usage patterns
+- **Resource Management**: Janitor automatically cleans up idle browsers
+
+## 🧹 Janitor System: Automatic Cleanup
+
+**The Problem:** Long-running crawlers accumulate idle browsers and consume memory over time.
+
+**My Solution:** An automatic janitor system that monitors and cleans up idle resources.
+
+```python
+async def monitor_janitor_activity():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/logs/janitor?limit=5")
+        logs = response.json()
+
+        print("Recent Cleanup Activities:")
+        for log in logs:
+            print(f"  {log['timestamp']}: {log['message']}")
+
+# Example output:
+# 2025-11-14 10:30:00: Cleaned up 2 cold pool browsers (idle > 5min)
+# 2025-11-14 10:25:00: Browser reuse rate: 85.3%
+# 2025-11-14 10:20:00: Hot pool browser promoted (10 requests)
+```
+
+## 🎮 Control Actions: Manual Management
+
+**The Problem:** Sometimes you need to manually intervene—kill a stuck browser, force cleanup, or restart resources.
+
+**My Solution:** Manual control actions via the API for operational troubleshooting.
+
+### Force Cleanup
+
+```python
+async def force_cleanup():
+    async with httpx.AsyncClient() as client:
+        response = await client.post("http://localhost:11235/monitor/actions/cleanup")
+        result = response.json()
+
+        print(f"Cleanup completed:")
+        print(f"  Browsers cleaned: {result.get('cleaned_count', 0)}")
+        print(f"  Memory freed: {result.get('memory_freed_mb', 0):.1f} MB")
+```
+
+### Kill Specific Browser
+
+```python
+async def kill_stuck_browser(browser_id: str):
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            "http://localhost:11235/monitor/actions/kill_browser",
+            json={"browser_id": browser_id}
+        )
+
+        if response.status_code == 200:
+            print(f"✅ Browser {browser_id} killed successfully")
+```
+
+### Reset Statistics
+
+```python
+async def reset_stats():
+    async with httpx.AsyncClient() as client:
+        response = await client.post("http://localhost:11235/monitor/stats/reset")
+        print("📊 Statistics reset for fresh monitoring")
+```
+
+## 📈 Production Integration Patterns
+
+### Prometheus Integration
+
+```python
+# Export metrics for Prometheus scraping
+async def export_prometheus_metrics():
+    async with httpx.AsyncClient() as client:
+        health = await client.get("http://localhost:11235/monitor/health")
+        data = health.json()
+
+        # Export in Prometheus format
+        metrics = f"""
+# HELP crawl4ai_memory_usage_percent Memory usage percentage
+# TYPE crawl4ai_memory_usage_percent gauge
+crawl4ai_memory_usage_percent {data['container']['memory_percent']}
+
+# HELP crawl4ai_request_success_rate Request success rate
+# TYPE crawl4ai_request_success_rate gauge
+crawl4ai_request_success_rate {data['stats']['success_rate_percent']}
+
+# HELP crawl4ai_browser_pool_count Total browsers in pool
+# TYPE crawl4ai_browser_pool_count gauge
+crawl4ai_browser_pool_count {data['pool']['permanent']['active'] + data['pool']['hot']['count'] + data['pool']['cold']['count']}
+"""
+        return metrics
+```
+
+### Alerting Example
+
+```python
+async def check_alerts():
+    async with httpx.AsyncClient() as client:
+        health = await client.get("http://localhost:11235/monitor/health")
+        data = health.json()
+
+        # Memory alert
+        if data['container']['memory_percent'] > 80:
+            print("🚨 ALERT: Memory usage above 80%")
+            # Trigger cleanup
+            await client.post("http://localhost:11235/monitor/actions/cleanup")
+
+        # Success rate alert
+        if data['stats']['success_rate_percent'] < 90:
+            print("🚨 ALERT: Success rate below 90%")
+            # Check error logs
+            errors = await client.get("http://localhost:11235/monitor/logs/errors")
+            print(f"Recent errors: {len(errors.json())}")
+
+        # Latency alert
+        if data['stats']['avg_latency_ms'] > 5000:
+            print("🚨 ALERT: Average latency above 5s")
+```
+
+### Key Metrics to Track
+
+```python
+CRITICAL_METRICS = {
+    "memory_usage": {
+        "current": "container.memory_percent",
+        "target": "<80%",
+        "alert_threshold": ">80%",
+        "action": "Force cleanup or scale"
+    },
+    "success_rate": {
+        "current": "stats.success_rate_percent",
+        "target": ">95%",
+        "alert_threshold": "<90%",
+        "action": "Check error logs"
+    },
+    "avg_latency": {
+        "current": "stats.avg_latency_ms",
+        "target": "<2000ms",
+        "alert_threshold": ">5000ms",
+        "action": "Investigate slow requests"
+    },
+    "browser_reuse_rate": {
+        "current": "browsers.summary.reuse_rate_percent",
+        "target": ">80%",
+        "alert_threshold": "<60%",
+        "action": "Check pool configuration"
+    },
+    "total_browsers": {
+        "current": "browsers.summary.total_count",
+        "target": "<15",
+        "alert_threshold": ">20",
+        "action": "Check for browser leaks"
+    },
+    "error_frequency": {
+        "current": "len(errors)",
+        "target": "<5/hour",
+        "alert_threshold": ">10/hour",
+        "action": "Review error patterns"
+    }
+}
+```
+
+## 🐛 Critical Bug Fixes
+
+This release includes significant bug fixes that improve stability and performance:
+
+### Async LLM Extraction (#1590)
+
+**The Problem:** LLM extraction was blocking async execution, causing URLs to be processed sequentially instead of in parallel (issue #1055).
+
+**The Fix:** Resolved the blocking issue to enable true parallel processing for LLM extraction.
+
+```python
+# Before v0.7.7: Sequential processing
+# After v0.7.7: True parallel processing
+
+async with AsyncWebCrawler() as crawler:
+    urls = ["url1", "url2", "url3", "url4"]
+
+    # Now processes truly in parallel with LLM extraction
+    results = await crawler.arun_many(
+        urls,
+        config=CrawlerRunConfig(
+            extraction_strategy=LLMExtractionStrategy(...)
+        )
+    )
+    # 4x faster for parallel LLM extraction!
+```
+
+**Expected Impact:** Major performance improvement for batch LLM extraction workflows.
+
+### DFS Deep Crawling (#1607)
+
+**The Problem:** DFS (Depth-First Search) deep crawl strategy had implementation issues.
+
+**The Fix:** Enhanced DFSDeepCrawlStrategy with proper seen URL tracking and improved documentation.
+
+### Browser & Crawler Config Documentation (#1609)
+
+**The Problem:** Documentation didn't match the actual `async_configs.py` implementation.
+
+**The Fix:** Updated all configuration documentation to accurately reflect the current implementation.
+
+### Sitemap Seeder (#1598)
+
+**The Problem:** Sitemap parsing and URL normalization issues in AsyncUrlSeeder (issue #1559).
+
+**The Fix:** Added comprehensive tests and fixes for sitemap namespace parsing and URL normalization.
+
+### Remove Overlay Elements (#1529)
+
+**The Problem:** The `remove_overlay_elements` functionality wasn't working (issue #1396).
+
+**The Fix:** Fixed by properly calling the injected JavaScript function.
+
+### Viewport Configuration (#1495)
+
+**The Problem:** Viewport configuration wasn't working in managed browsers (issue #1490).
+
+**The Fix:** Added proper viewport size configuration support for browser launch.
+
+### Managed Browser CDP Timing (#1528)
+
+**The Problem:** CDP (Chrome DevTools Protocol) endpoint verification had timing issues causing connection failures (issue #1445).
+
+**The Fix:** Added exponential backoff for CDP endpoint verification to handle timing variations.
+
+### Security Updates
+
+- **pyOpenSSL**: Updated from >=24.3.0 to >=25.3.0 to address security vulnerability
+- Added verification tests for the security update
+
+### Docker Fixes
+
+- **Port Standardization**: Fixed inconsistent port usage (11234 vs 11235) - now standardized to 11235
+- **LLM Environment**: Fixed LLM API key handling for multi-provider support (PR #1537)
+- **Error Handling**: Improved Docker API error messages with comprehensive status codes
+- **Serialization**: Fixed `fit_html` property serialization in `/crawl` and `/crawl/stream` endpoints
+
+### Other Important Fixes
+
+- **arun_many Returns**: Fixed function to always return a list, even on exception (PR #1530)
+- **Webhook Serialization**: Properly serialize Pydantic HttpUrl in webhook config
+- **LLMConfig Documentation**: Fixed casing and variable name consistency (issue #1551)
+- **Python Version**: Dropped Python 3.9 support, now requires Python >=3.10
+
+## 📊 Expected Real-World Impact
+
+### For DevOps & Infrastructure Teams
+- **Full Visibility**: Know exactly what's happening inside your crawling infrastructure
+- **Proactive Monitoring**: Catch issues before they become problems
+- **Resource Optimization**: Identify memory leaks and performance bottlenecks
+- **Operational Control**: Manual intervention when automated systems need help
+
+### For Production Deployments
+- **Enterprise Observability**: Prometheus, Grafana, and alerting integration
+- **Debugging**: Real-time logs and error tracking
+- **Capacity Planning**: Historical metrics for scaling decisions
+- **SLA Monitoring**: Track success rates and latency against targets
+
+### For Development Teams
+- **Local Monitoring**: Understand crawler behavior during development
+- **Performance Testing**: Measure impact of configuration changes
+- **Troubleshooting**: Quickly identify and fix issues
+- **Learning**: See exactly how the browser pool works
+
+## 🔄 Breaking Changes
+
+**None!** This release is fully backward compatible.
+
+- All existing Docker configurations continue to work
+- No API changes to existing endpoints
+- Monitoring is additive functionality
+- No migration required
+
+## 🚀 Upgrade Instructions
+
+### Docker
+
+```bash
+# Pull the latest version
+docker pull unclecode/crawl4ai:0.7.7
+
+# Or use the latest tag
+docker pull unclecode/crawl4ai:latest
+
+# Run with monitoring enabled (default)
+docker run -d \
+  -p 11235:11235 \
+  --shm-size=1g \
+  --name crawl4ai \
+  unclecode/crawl4ai:0.7.7
+
+# Access the monitoring dashboard
+open http://localhost:11235/dashboard
+```
+
+### Python Package
+
+```bash
+# Upgrade to latest version
+pip install --upgrade crawl4ai
+
+# Or install specific version
+pip install crawl4ai==0.7.7
+```
+
+## 🎬 Try the Demo
+
+Run the comprehensive demo that showcases all monitoring features:
+
+```bash
+python docs/releases_review/demo_v0.7.7.py
+```
+
+**The demo includes:**
+1. System health overview with live metrics
+2. Request tracking with active/completed monitoring
+3. Browser pool management (permanent/hot/cold)
+4. Complete Monitor API endpoint examples
+5. WebSocket streaming demonstration
+6. Control actions (cleanup, kill, restart)
+7. Production metrics and alerting patterns
+8. Self-hosting value proposition
+
+## 📚 Documentation
+
+### New Documentation
+- **[Self-Hosting Guide](https://docs.crawl4ai.com/core/self-hosting/)** - Complete self-hosting documentation with monitoring
+- **Demo Script**: `docs/releases_review/demo_v0.7.7.py` - Working examples
+
+### Updated Documentation
+- **Docker Deployment** → **Self-Hosting** (renamed for better positioning)
+- Added comprehensive monitoring sections
+- Production integration patterns
+- WebSocket streaming examples
+
+## 💡 Pro Tips
+
+1. **Start with the dashboard** - Visit `/dashboard` to get familiar with the monitoring system
+2. **Track the 6 key metrics** - Memory, success rate, latency, reuse rate, browser count, errors
+3. **Set up alerting early** - Use the Monitor API to build alerts before issues occur
+4. **Monitor browser pool efficiency** - Aim for >80% reuse rate for optimal performance
+5. **Use WebSocket for custom dashboards** - Build tailored monitoring UIs for your team
+6. **Leverage Prometheus integration** - Export metrics for long-term storage and analysis
+7. **Check janitor logs** - Understand automatic cleanup patterns
+8. **Use control actions judiciously** - Manual interventions are for exceptional cases
+
+## 🙏 Acknowledgments
+
+Thank you to our community for the feedback, bug reports, and feature requests that shaped this release. Special thanks to everyone who contributed to the issues that were fixed in this version.
+
+The monitoring system was built based on real user needs for production deployments, and your input made it comprehensive and practical.
+
+## 📞 Support & Resources
+
+- **📖 Documentation**: [docs.crawl4ai.com](https://docs.crawl4ai.com)
+- **🐙 GitHub**: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
+- **💬 Discord**: [discord.gg/crawl4ai](https://discord.gg/jP8KfhDhyN)
+- **🐦 Twitter**: [@unclecode](https://x.com/unclecode)
+- **📊 Dashboard**: `http://localhost:11235/dashboard` (when running)
+
+---
+
+**Crawl4AI v0.7.7 delivers complete self-hosting with enterprise-grade monitoring. You now have full visibility and control over your web crawling infrastructure. The monitoring dashboard, comprehensive API, and WebSocket streaming give you everything needed for production deployments. Try the self-hosting platform—it's a game changer for operational excellence!**
+
+**Happy crawling with full visibility!** 🕷️📊
+
+*- unclecode*

docs/blog/release-v0.7.8.md

@@ -0,0 +1,327 @@
+# Crawl4AI v0.7.8: Stability & Bug Fix Release
+
+*December 2025*
+
+---
+
+I'm releasing Crawl4AI v0.7.8—a focused stability release that addresses 11 bugs reported by the community. While there are no new features in this release, these fixes resolve important issues affecting Docker deployments, LLM extraction, URL handling, and dependency compatibility.
+
+## What's Fixed at a Glance
+
+- **Docker API**: Fixed ContentRelevanceFilter deserialization, ProxyConfig serialization, and cache folder permissions
+- **LLM Extraction**: Configurable rate limiter backoff, HTML input format support, and proper URL handling for raw HTML
+- **URL Handling**: Correct relative URL resolution after JavaScript redirects
+- **Dependencies**: Replaced deprecated PyPDF2 with pypdf, Pydantic v2 ConfigDict compatibility
+- **AdaptiveCrawler**: Fixed query expansion to actually use LLM instead of hardcoded mock data
+
+## Bug Fixes
+
+### Docker & API Fixes
+
+#### ContentRelevanceFilter Deserialization (#1642)
+
+**The Problem:** When sending deep crawl requests to the Docker API with `ContentRelevanceFilter`, the server failed to deserialize the filter, causing requests to fail.
+
+**The Fix:** I added `ContentRelevanceFilter` to the public exports and enhanced the deserialization logic with dynamic imports.
+
+```python
+# This now works correctly in Docker API
+import httpx
+
+request = {
+    "urls": ["https://docs.example.com"],
+    "crawler_config": {
+        "deep_crawl_strategy": {
+            "type": "BFSDeepCrawlStrategy",
+            "max_depth": 2,
+            "filter_chain": [
+                {
+                    "type": "ContentRelevanceFilter",
+                    "query": "API documentation",
+                    "threshold": 0.3
+                }
+            ]
+        }
+    }
+}
+
+async with httpx.AsyncClient() as client:
+    response = await client.post("http://localhost:11235/crawl", json=request)
+    # Previously failed, now works!
+```
+
+#### ProxyConfig JSON Serialization (#1629)
+
+**The Problem:** `BrowserConfig.to_dict()` failed when `proxy_config` was set because `ProxyConfig` wasn't being serialized to a dictionary.
+
+**The Fix:** `ProxyConfig.to_dict()` is now called during serialization.
+
+```python
+from crawl4ai import BrowserConfig
+from crawl4ai.async_configs import ProxyConfig
+
+proxy = ProxyConfig(
+    server="http://proxy.example.com:8080",
+    username="user",
+    password="pass"
+)
+
+config = BrowserConfig(headless=True, proxy_config=proxy)
+
+# Previously raised TypeError, now works
+config_dict = config.to_dict()
+json.dumps(config_dict)  # Valid JSON
+```
+
+#### Docker Cache Folder Permissions (#1638)
+
+**The Problem:** The `.cache` folder in the Docker image had incorrect permissions, causing crawling to fail when caching was enabled.
+
+**The Fix:** Corrected ownership and permissions during image build.
+
+```bash
+# Cache now works correctly in Docker
+docker run -d -p 11235:11235 \
+    --shm-size=1g \
+    -v ./my-cache:/app/.cache \
+    unclecode/crawl4ai:0.7.8
+```
+
+---
+
+### LLM & Extraction Fixes
+
+#### Configurable Rate Limiter Backoff (#1269)
+
+**The Problem:** The LLM rate limiting backoff parameters were hardcoded, making it impossible to adjust retry behavior for different API rate limits.
+
+**The Fix:** `LLMConfig` now accepts three new parameters for complete control over retry behavior.
+
+```python
+from crawl4ai import LLMConfig
+
+# Default behavior (unchanged)
+default_config = LLMConfig(provider="openai/gpt-4o-mini")
+# backoff_base_delay=2, backoff_max_attempts=3, backoff_exponential_factor=2
+
+# Custom configuration for APIs with strict rate limits
+custom_config = LLMConfig(
+    provider="openai/gpt-4o-mini",
+    backoff_base_delay=5,           # Wait 5 seconds on first retry
+    backoff_max_attempts=5,          # Try up to 5 times
+    backoff_exponential_factor=3     # Multiply delay by 3 each attempt
+)
+
+# Retry sequence: 5s -> 15s -> 45s -> 135s -> 405s
+```
+
+#### LLM Strategy HTML Input Support (#1178)
+
+**The Problem:** `LLMExtractionStrategy` always sent markdown to the LLM, but some extraction tasks work better with HTML structure preserved.
+
+**The Fix:** Added `input_format` parameter supporting `"markdown"`, `"html"`, `"fit_markdown"`, `"cleaned_html"`, and `"fit_html"`.
+
+```python
+from crawl4ai import LLMExtractionStrategy, LLMConfig
+
+# Default: markdown input (unchanged)
+markdown_strategy = LLMExtractionStrategy(
+    llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+    instruction="Extract product information"
+)
+
+# NEW: HTML input - preserves table/list structure
+html_strategy = LLMExtractionStrategy(
+    llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+    instruction="Extract the data table preserving structure",
+    input_format="html"
+)
+
+# NEW: Filtered markdown - only relevant content
+fit_strategy = LLMExtractionStrategy(
+    llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+    instruction="Summarize the main content",
+    input_format="fit_markdown"
+)
+```
+
+#### Raw HTML URL Variable (#1116)
+
+**The Problem:** When using `url="raw:<html>..."`, the entire HTML content was being passed to extraction strategies as the URL parameter, polluting LLM prompts.
+
+**The Fix:** The URL is now correctly set to `"Raw HTML"` for raw HTML inputs.
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+html = "<html><body><h1>Test</h1></body></html>"
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url=f"raw:{html}",
+        config=CrawlerRunConfig(extraction_strategy=my_strategy)
+    )
+    # extraction_strategy receives url="Raw HTML" instead of the HTML blob
+```
+
+---
+
+### URL Handling Fix
+
+#### Relative URLs After Redirects (#1268)
+
+**The Problem:** When JavaScript caused a page redirect, relative links were resolved against the original URL instead of the final URL.
+
+**The Fix:** `redirected_url` now captures the actual page URL after all JavaScript execution completes.
+
+```python
+from crawl4ai import AsyncWebCrawler
+
+async with AsyncWebCrawler() as crawler:
+    # Page at /old-page redirects via JS to /new-page
+    result = await crawler.arun(url="https://example.com/old-page")
+
+    # BEFORE: redirected_url = "https://example.com/old-page"
+    # AFTER:  redirected_url = "https://example.com/new-page"
+
+    # Links are now correctly resolved against the final URL
+    for link in result.links['internal']:
+        print(link['href'])  # Relative links resolved correctly
+```
+
+---
+
+### Dependency & Compatibility Fixes
+
+#### PyPDF2 Replaced with pypdf (#1412)
+
+**The Problem:** PyPDF2 was deprecated in 2022 and is no longer maintained.
+
+**The Fix:** Replaced with the actively maintained `pypdf` library.
+
+```python
+# Installation (unchanged)
+pip install crawl4ai[pdf]
+
+# The PDF processor now uses pypdf internally
+# No code changes required - API remains the same
+```
+
+#### Pydantic v2 ConfigDict Compatibility (#678)
+
+**The Problem:** Using the deprecated `class Config` syntax caused deprecation warnings with Pydantic v2.
+
+**The Fix:** Migrated to `model_config = ConfigDict(...)` syntax.
+
+```python
+# No more deprecation warnings when importing crawl4ai models
+from crawl4ai.models import CrawlResult
+from crawl4ai import CrawlerRunConfig, BrowserConfig
+
+# All models are now Pydantic v2 compatible
+```
+
+---
+
+### AdaptiveCrawler Fix
+
+#### Query Expansion Using LLM (#1621)
+
+**The Problem:** The `EmbeddingStrategy` in AdaptiveCrawler had commented-out LLM code and was using hardcoded mock query variations instead.
+
+**The Fix:** Uncommented and activated the LLM call for actual query expansion.
+
+```python
+# AdaptiveCrawler query expansion now actually uses the LLM
+# Instead of hardcoded variations like:
+# variations = {'queries': ['what are the best vegetables...']}
+
+# The LLM generates relevant query variations based on your actual query
+```
+
+---
+
+### Code Formatting Fix
+
+#### Import Statement Formatting (#1181)
+
+**The Problem:** When extracting code from web pages, import statements were sometimes concatenated without proper line separation.
+
+**The Fix:** Import statements now maintain proper newline separation.
+
+```python
+# BEFORE: "import osimport sysfrom pathlib import Path"
+# AFTER:
+# import os
+# import sys
+# from pathlib import Path
+```
+
+---
+
+## Breaking Changes
+
+**None!** This release is fully backward compatible.
+
+- All existing code continues to work without modification
+- New parameters have sensible defaults matching previous behavior
+- No API changes to existing functionality
+
+---
+
+## Upgrade Instructions
+
+### Python Package
+
+```bash
+pip install --upgrade crawl4ai
+# or
+pip install crawl4ai==0.7.8
+```
+
+### Docker
+
+```bash
+# Pull the latest version
+docker pull unclecode/crawl4ai:0.7.8
+
+# Run
+docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.8
+```
+
+---
+
+## Verification
+
+Run the verification tests to confirm all fixes are working:
+
+```bash
+python docs/releases_review/demo_v0.7.8.py
+```
+
+This runs actual tests that verify each bug fix is properly implemented.
+
+---
+
+## Acknowledgments
+
+Thank you to everyone who reported these issues and provided detailed reproduction steps. Your bug reports make Crawl4AI better for everyone.
+
+Issues fixed: #1642, #1638, #1629, #1621, #1412, #1269, #1268, #1181, #1178, #1116, #678
+
+---
+
+## Support & Resources
+
+- **Documentation**: [docs.crawl4ai.com](https://docs.crawl4ai.com)
+- **GitHub**: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
+- **Discord**: [discord.gg/crawl4ai](https://discord.gg/jP8KfhDhyN)
+- **Twitter**: [@unclecode](https://x.com/unclecode)
+
+---
+
+**This stability release ensures Crawl4AI works reliably across Docker deployments, LLM extraction workflows, and various edge cases. Thank you for your continued support and feedback!**
+
+**Happy crawling!**
+
+*- unclecode*

docs/blog/release-v0.8.0.md

@@ -0,0 +1,243 @@
+# Crawl4AI v0.8.0 Release Notes
+
+**Release Date**: January 2026
+**Previous Version**: v0.7.6
+**Status**: Release Candidate
+
+---
+
+## Highlights
+
+- **Critical Security Fixes** for Docker API deployment
+- **11 New Features** including crash recovery, prefetch mode, and proxy improvements
+- **Breaking Changes** - see migration guide below
+
+---
+
+## Breaking Changes
+
+### 1. Docker API: Hooks Disabled by Default
+
+**What changed**: Hooks are now disabled by default on the Docker API.
+
+**Why**: Security fix for Remote Code Execution (RCE) vulnerability.
+
+**Who is affected**: Users of the Docker API who use the `hooks` parameter in `/crawl` requests.
+
+**Migration**:
+```bash
+# To re-enable hooks (only if you trust all API users):
+export CRAWL4AI_HOOKS_ENABLED=true
+```
+
+### 2. Docker API: file:// URLs Blocked
+
+**What changed**: The endpoints `/execute_js`, `/screenshot`, `/pdf`, and `/html` now reject `file://` URLs.
+
+**Why**: Security fix for Local File Inclusion (LFI) vulnerability.
+
+**Who is affected**: Users who were reading local files via the Docker API.
+
+**Migration**: Use the Python library directly for local file processing:
+```python
+# Instead of API call with file:// URL, use library:
+from crawl4ai import AsyncWebCrawler
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(url="file:///path/to/file.html")
+```
+
+---
+
+## Security Fixes
+
+### Critical: Remote Code Execution via Hooks (CVE Pending)
+
+**Severity**: CRITICAL (CVSS 10.0)
+**Affected**: Docker API deployment (all versions before v0.8.0)
+**Vector**: `POST /crawl` with malicious `hooks` parameter
+
+**Details**: The `__import__` builtin was available in hook code, allowing attackers to import `os`, `subprocess`, etc. and execute arbitrary commands.
+
+**Fix**:
+1. Removed `__import__` from allowed builtins
+2. Hooks disabled by default (`CRAWL4AI_HOOKS_ENABLED=false`)
+
+### High: Local File Inclusion via file:// URLs (CVE Pending)
+
+**Severity**: HIGH (CVSS 8.6)
+**Affected**: Docker API deployment (all versions before v0.8.0)
+**Vector**: `POST /execute_js` (and other endpoints) with `file:///etc/passwd`
+
+**Details**: API endpoints accepted `file://` URLs, allowing attackers to read arbitrary files from the server.
+
+**Fix**: URL scheme validation now only allows `http://`, `https://`, and `raw:` URLs.
+
+### Credits
+
+Discovered by **Neo by ProjectDiscovery** ([projectdiscovery.io](https://projectdiscovery.io)) - December 2025
+
+---
+
+## New Features
+
+### 1. init_scripts Support for BrowserConfig
+
+Pre-page-load JavaScript injection for stealth evasions.
+
+```python
+config = BrowserConfig(
+    init_scripts=[
+        "Object.defineProperty(navigator, 'webdriver', {get: () => false})"
+    ]
+)
+```
+
+### 2. CDP Connection Improvements
+
+- WebSocket URL support (`ws://`, `wss://`)
+- Proper cleanup with `cdp_cleanup_on_close=True`
+- Browser reuse across multiple connections
+
+### 3. Crash Recovery for Deep Crawl Strategies
+
+All deep crawl strategies (BFS, DFS, Best-First) now support crash recovery:
+
+```python
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+strategy = BFSDeepCrawlStrategy(
+    max_depth=3,
+    resume_state=saved_state,  # Resume from checkpoint
+    on_state_change=save_callback  # Persist state in real-time
+)
+```
+
+### 4. PDF and MHTML for raw:/file:// URLs
+
+Generate PDFs and MHTML from cached HTML content.
+
+### 5. Screenshots for raw:/file:// URLs
+
+Render cached HTML and capture screenshots.
+
+### 6. base_url Parameter for CrawlerRunConfig
+
+Proper URL resolution for raw: HTML processing:
+
+```python
+config = CrawlerRunConfig(base_url='https://example.com')
+result = await crawler.arun(url='raw:{html}', config=config)
+```
+
+### 7. Prefetch Mode for Two-Phase Deep Crawling
+
+Fast link extraction without full page processing:
+
+```python
+config = CrawlerRunConfig(prefetch=True)
+```
+
+### 8. Proxy Rotation and Configuration
+
+Enhanced proxy rotation with sticky sessions support.
+
+### 9. Proxy Support for HTTP Strategy
+
+Non-browser crawler now supports proxies.
+
+### 10. Browser Pipeline for raw:/file:// URLs
+
+New `process_in_browser` parameter for browser operations on local content:
+
+```python
+config = CrawlerRunConfig(
+    process_in_browser=True,  # Force browser processing
+    screenshot=True
+)
+result = await crawler.arun(url='raw:<html>...</html>', config=config)
+```
+
+### 11. Smart TTL Cache for Sitemap URL Seeder
+
+Intelligent cache invalidation for sitemaps:
+
+```python
+config = SeedingConfig(
+    cache_ttl_hours=24,
+    validate_sitemap_lastmod=True
+)
+```
+
+---
+
+## Bug Fixes
+
+### raw: URL Parsing Truncates at # Character
+
+**Problem**: CSS color codes like `#eee` were being truncated.
+
+**Before**: `raw:body{background:#eee}` → `body{background:`
+**After**: `raw:body{background:#eee}` → `body{background:#eee}`
+
+### Caching System Improvements
+
+Various fixes to cache validation and persistence.
+
+---
+
+## Documentation Updates
+
+- Multi-sample schema generation documentation
+- URL seeder smart TTL cache parameters
+- Security documentation (SECURITY.md)
+
+---
+
+## Upgrade Guide
+
+### From v0.7.x to v0.8.0
+
+1. **Update the package**:
+   ```bash
+   pip install --upgrade crawl4ai
+   ```
+
+2. **Docker API users**:
+   - Hooks are now disabled by default
+   - If you need hooks: `export CRAWL4AI_HOOKS_ENABLED=true`
+   - `file://` URLs no longer work on API (use library directly)
+
+3. **Review security settings**:
+   ```yaml
+   # config.yml - recommended for production
+   security:
+     enabled: true
+     jwt_enabled: true
+   ```
+
+4. **Test your integration** before deploying to production
+
+### Breaking Change Checklist
+
+- [ ] Check if you use `hooks` parameter in API calls
+- [ ] Check if you use `file://` URLs via the API
+- [ ] Update environment variables if needed
+- [ ] Review security configuration
+
+---
+
+## Full Changelog
+
+See [CHANGELOG.md](../CHANGELOG.md) for complete version history.
+
+---
+
+## Contributors
+
+Thanks to all contributors who made this release possible.
+
+Special thanks to **Neo by ProjectDiscovery** for responsible security disclosure.
+
+---
+
+*For questions or issues, please open a [GitHub Issue](https://github.com/unclecode/crawl4ai/issues).*

docs/examples/c4a_script/tutorial/README.md

@@ -18,7 +18,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip
 
 2. **Install Dependencies**
    ```bash
-   pip install flask
+   pip install -r requirements.txt
    ```
 
 3. **Launch the Server**
@@ -28,7 +28,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip
 
 4. **Open in Browser**
    ```
-   http://localhost:8080
+   http://localhost:8000
    ```
 
 **🌐 Try Online**: [Live Demo](https://docs.crawl4ai.com/c4a-script/demo)
@@ -325,7 +325,7 @@ Powers the recording functionality:
 ### Configuration
 ```python
 # server.py configuration
-PORT = 8080
+PORT = 8000
 DEBUG = True
 THREADED = True
 ```
@@ -343,9 +343,9 @@ THREADED = True
 **Port Already in Use**
 ```bash
 # Kill existing process
-lsof -ti:8080 | xargs kill -9
+lsof -ti:8000 | xargs kill -9
 # Or use different port
-python server.py --port 8081
+python server.py --port 8001
 ```
 
 **Blockly Not Loading**

docs/examples/c4a_script/tutorial/server.py

@@ -216,7 +216,7 @@ def get_examples():
             'name': 'Handle Cookie Banner',
             'description': 'Accept cookies and close newsletter popup',
             'script': '''# Handle cookie banner and newsletter
-GO http://127.0.0.1:8080/playground/
+GO http://127.0.0.1:8000/playground/
 WAIT `body` 2
 IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
 IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`'''

docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_aws_waf.py

@@ -0,0 +1,62 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/awsWaf/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"              # your api key of capsolver
+site_url = "https://nft.porsche.com/onboarding@6"  # page url of your target site
+cookie_domain = ".nft.porsche.com"                 # the domain name to which you want to apply the cookie
+captcha_type = "AntiAwsWafTaskProxyLess"           # type of your target captcha
+capsolver.api_key = api_key
+
+
+async def main():
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # get aws waf cookie using capsolver sdk
+        solution = capsolver.solve({
+            "type": captcha_type,
+            "websiteURL": site_url,
+        })
+        cookie = solution["cookie"]
+        print("aws waf cookie:", cookie)
+
+        js_code = """
+            document.cookie = \'aws-waf-token=""" + cookie + """;domain=""" + cookie_domain + """;path=/\';
+            location.reload();
+        """
+
+        wait_condition = """() => {
+            return document.title === \'Join Porsche’s journey into Web3\';
+        }"""
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test",
+            js_code=js_code,
+            js_only=True,
+            wait_for=f"js:{wait_condition}"
+        )
+
+        result_next = await crawler.arun(
+            url=site_url,
+            config=run_config,
+        )
+        print(result_next.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_challenge.py

@@ -0,0 +1,60 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/cloudflare_challenge/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"          # your api key of capsolver
+site_url = "https://gitlab.com/users/sign_in"  # page url of your target site
+captcha_type = "AntiCloudflareTask"            # type of your target captcha
+# your http proxy to solve cloudflare challenge
+proxy_server = "proxy.example.com:8080"
+proxy_username = "myuser"
+proxy_password = "mypass"
+capsolver.api_key = api_key
+
+
+async def main():
+    # get challenge cookie using capsolver sdk
+    solution = capsolver.solve({
+        "type": captcha_type,
+        "websiteURL": site_url,
+        "proxy": f"{proxy_server}:{proxy_username}:{proxy_password}",
+    })
+    cookies = solution["cookies"]
+    user_agent = solution["userAgent"]
+    print("challenge cookies:", cookies)
+
+    cookies_list = []
+    for name, value in cookies.items():
+        cookies_list.append({
+            "name": name,
+            "value": value,
+            "url": site_url,
+        })
+
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+        user_agent=user_agent,
+        cookies=cookies_list,
+        proxy_config={
+            "server": f"http://{proxy_server}",
+            "username": proxy_username,
+            "password": proxy_password,
+        },
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+        print(result.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_cloudflare_turnstile.py

@@ -0,0 +1,64 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/cloudflare_turnstile/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"                       # your api key of capsolver
+site_key = "0x4AAAAAAAGlwMzq_9z6S9Mh"                       # site key of your target site
+site_url = "https://clifford.io/demo/cloudflare-turnstile"  # page url of your target site
+captcha_type = "AntiTurnstileTaskProxyLess"                 # type of your target captcha
+capsolver.api_key = api_key
+
+
+async def main():
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # get turnstile token using capsolver sdk
+        solution = capsolver.solve({
+            "type": captcha_type,
+            "websiteURL": site_url,
+            "websiteKey": site_key,
+        })
+        token = solution["token"]
+        print("turnstile token:", token)
+
+        js_code = """
+            document.querySelector(\'input[name="cf-turnstile-response"]\').value = \'"""+token+"""\';
+            document.querySelector(\'button[type="submit"]\').click();
+        """
+
+        wait_condition = """() => {
+            const items = document.querySelectorAll(\'h1\');
+            return items.length === 0;
+        }"""
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test",
+            js_code=js_code,
+            js_only=True,
+            wait_for=f"js:{wait_condition}"
+        )
+
+        result_next = await crawler.arun(
+            url=site_url,
+            config=run_config,
+        )
+        print(result_next.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v2.py

@@ -0,0 +1,67 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/ReCaptchaV2/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"                                      # your api key of capsolver
+site_key = "6LfW6wATAAAAAHLqO2pb8bDBahxlMxNdo9g947u9"                      # site key of your target site
+site_url = "https://recaptcha-demo.appspot.com/recaptcha-v2-checkbox.php"  # page url of your target site
+captcha_type = "ReCaptchaV2TaskProxyLess"                                  # type of your target captcha
+capsolver.api_key = api_key
+
+
+async def main():
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+    )
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # get recaptcha token using capsolver sdk
+        solution = capsolver.solve({
+            "type": captcha_type,
+            "websiteURL": site_url,
+            "websiteKey": site_key,
+        })
+        token = solution["gRecaptchaResponse"]
+        print("recaptcha token:", token)
+
+        js_code = """
+            const textarea = document.getElementById(\'g-recaptcha-response\');
+            if (textarea) {
+                textarea.value = \"""" + token + """\";
+                document.querySelector(\'button.form-field[type="submit"]\').click();
+            }
+        """
+
+        wait_condition = """() => {
+            const items = document.querySelectorAll(\'h2\');
+            return items.length > 1;
+        }"""
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test",
+            js_code=js_code,
+            js_only=True,
+            wait_for=f"js:{wait_condition}"
+        )
+
+        result_next = await crawler.arun(
+            url=site_url,
+            config=run_config,
+        )
+        print(result_next.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_api_integration/solve_recaptcha_v3.py

@@ -0,0 +1,75 @@
+import asyncio
+import capsolver
+from crawl4ai import *
+
+
+# TODO: set your config
+# Docs: https://docs.capsolver.com/guide/captcha/ReCaptchaV3/
+api_key = "CAP-xxxxxxxxxxxxxxxxxxxxx"                                            # your api key of capsolver
+site_key = "6LdKlZEpAAAAAAOQjzC2v_d36tWxCl6dWsozdSy9"                            # site key of your target site
+site_url = "https://recaptcha-demo.appspot.com/recaptcha-v3-request-scores.php"  # page url of your target site
+page_action = "examples/v3scores"                                                # page action of your target site
+captcha_type = "ReCaptchaV3TaskProxyLess"                                        # type of your target captcha
+capsolver.api_key = api_key
+
+
+async def main():
+    browser_config = BrowserConfig(
+        verbose=True,
+        headless=False,
+        use_persistent_context=True,
+    )
+
+    # get recaptcha token using capsolver sdk
+    solution = capsolver.solve({
+        "type": captcha_type,
+        "websiteURL": site_url,
+        "websiteKey": site_key,
+        "pageAction": page_action,
+    })
+    token = solution["gRecaptchaResponse"]
+    print("recaptcha token:", token)
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        await crawler.arun(
+            url=site_url,
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        js_code = """
+            const originalFetch = window.fetch;
+
+            window.fetch = function(...args) {
+              if (typeof args[0] === 'string' && args[0].includes('/recaptcha-v3-verify.php')) {
+                const url = new URL(args[0], window.location.origin);
+                url.searchParams.set('action', '""" + token + """');
+                args[0] = url.toString();
+                document.querySelector('.token').innerHTML = "fetch('/recaptcha-v3-verify.php?action=examples/v3scores&token=""" + token + """')";
+                console.log('Fetch URL hooked:', args[0]);
+              }
+              return originalFetch.apply(this, args);
+            };
+        """
+
+        wait_condition = """() => {
+            return document.querySelector('.step3:not(.hidden)');
+        }"""
+
+        run_config = CrawlerRunConfig(
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test",
+            js_code=js_code,
+            js_only=True,
+            wait_for=f"js:{wait_condition}"
+        )
+
+        result_next = await crawler.arun(
+            url=site_url,
+            config=run_config,
+        )
+        print(result_next.markdown)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_aws_waf.py

@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://nft.porsche.com/onboarding@6",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_challenge.py

@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://gitlab.com/users/sign_in",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_cloudflare_turnstile.py

@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://clifford.io/demo/cloudflare-turnstile",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v2.py

@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://recaptcha-demo.appspot.com/recaptcha-v2-checkbox.php",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/capsolver_captcha_solver/capsolver_extension_integration/solve_recaptcha_v3.py

@@ -0,0 +1,36 @@
+import time
+import asyncio
+from crawl4ai import *
+
+
+# TODO: the user data directory that includes the capsolver extension
+user_data_dir = "/browser-profile/Default1"
+
+"""
+The capsolver extension supports more features, such as:
+    - Telling the extension when to start solving captcha.
+    - Calling functions to check whether the captcha has been solved, etc.
+Reference blog: https://docs.capsolver.com/guide/automation-tool-integration/
+"""
+
+browser_config = BrowserConfig(
+    verbose=True,
+    headless=False,
+    user_data_dir=user_data_dir,
+    use_persistent_context=True,
+)
+
+async def main():
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result_initial = await crawler.arun(
+            url="https://recaptcha-demo.appspot.com/recaptcha-v3-request-scores.php",
+            cache_mode=CacheMode.BYPASS,
+            session_id="session_captcha_test"
+        )
+
+        # do something later
+        time.sleep(300)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/cloud_browser/scrapeless_browser.py

@@ -0,0 +1,61 @@
+import json
+import asyncio
+from urllib.parse import quote, urlencode
+from crawl4ai import CrawlerRunConfig, BrowserConfig, AsyncWebCrawler
+
+# Scrapeless provides a free anti-detection fingerprint browser client and cloud browsers:
+# https://www.scrapeless.com/en/blog/scrapeless-nstbrowser-strategic-integration
+
+async def main():
+    # customize browser fingerprint
+    fingerprint = {
+        "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.1.2.3 Safari/537.36",
+        "platform": "Windows",
+        "screen": {
+            "width": 1280, "height": 1024
+        },
+        "localization": {
+            "languages": ["zh-HK", "en-US", "en"], "timezone": "Asia/Hong_Kong",
+        }
+    }
+
+    fingerprint_json = json.dumps(fingerprint)
+    encoded_fingerprint = quote(fingerprint_json)
+
+    scrapeless_params = {
+        "token": "your token",
+        "sessionTTL": 1000,
+        "sessionName": "Demo",
+        "fingerprint": encoded_fingerprint,
+        # Sets the target country/region for the proxy, sending requests via an IP address from that region. You can specify a country code (e.g., US for the United States, GB for the United Kingdom, ANY for any country). See country codes for all supported options.
+        # "proxyCountry": "ANY",
+        # create profile on scrapeless
+        # "profileId": "your profileId",
+        # For more usage details, please refer to https://docs.scrapeless.com/en/scraping-browser/quickstart/getting-started
+    }
+    query_string = urlencode(scrapeless_params)
+    scrapeless_connection_url = f"wss://browser.scrapeless.com/api/v2/browser?{query_string}"
+    async with AsyncWebCrawler(
+        config=BrowserConfig(
+            headless=False,
+            browser_mode="cdp",
+            cdp_url=scrapeless_connection_url,
+        )
+    ) as crawler:
+        result = await crawler.arun(
+            url="https://www.scrapeless.com/en",
+            config=CrawlerRunConfig(
+                wait_for="css:.content",
+                scan_full_page=True,
+            ),
+        )
+        print("-" * 20)
+        print(f'Status Code: {result.status_code}')
+        print("-" * 20)
+        print(f'Title: {result.metadata["title"]}')
+        print(f'Description: {result.metadata["description"]}')
+        print("-" * 20)
+
+if __name__ == "__main__":
+    asyncio.run(main())
+    

docs/examples/deep_crawl_crash_recovery.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+"""
+Deep Crawl Crash Recovery Example
+
+This example demonstrates how to implement crash recovery for long-running
+deep crawls. The feature is useful for:
+
+- Cloud deployments with spot/preemptible instances
+- Long-running crawls that may be interrupted
+- Distributed crawling with state coordination
+
+Key concepts:
+- `on_state_change`: Callback fired after each URL is processed
+- `resume_state`: Pass saved state to continue from a checkpoint
+- `export_state()`: Get the last captured state manually
+
+Works with all strategies: BFSDeepCrawlStrategy, DFSDeepCrawlStrategy,
+BestFirstCrawlingStrategy
+"""
+
+import asyncio
+import json
+import os
+from pathlib import Path
+from typing import Dict, Any, List
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+
+# File to store crawl state (in production, use Redis/database)
+STATE_FILE = Path("crawl_state.json")
+
+
+async def save_state_to_file(state: Dict[str, Any]) -> None:
+    """
+    Callback to save state after each URL is processed.
+
+    In production, you might save to:
+    - Redis: await redis.set("crawl_state", json.dumps(state))
+    - Database: await db.execute("UPDATE crawls SET state = ?", json.dumps(state))
+    - S3: await s3.put_object(Bucket="crawls", Key="state.json", Body=json.dumps(state))
+    """
+    with open(STATE_FILE, "w") as f:
+        json.dump(state, f, indent=2)
+    print(f"  [State saved] Pages: {state['pages_crawled']}, Pending: {len(state['pending'])}")
+
+
+def load_state_from_file() -> Dict[str, Any] | None:
+    """Load previously saved state, if it exists."""
+    if STATE_FILE.exists():
+        with open(STATE_FILE, "r") as f:
+            return json.load(f)
+    return None
+
+
+async def example_basic_state_persistence():
+    """
+    Example 1: Basic state persistence with file storage.
+
+    The on_state_change callback is called after each URL is processed,
+    allowing you to save progress in real-time.
+    """
+    print("\n" + "=" * 60)
+    print("Example 1: Basic State Persistence")
+    print("=" * 60)
+
+    # Clean up any previous state
+    if STATE_FILE.exists():
+        STATE_FILE.unlink()
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=2,
+        max_pages=5,
+        on_state_change=save_state_to_file,  # Save after each URL
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    print("\nStarting crawl with state persistence...")
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        results = await crawler.arun("https://books.toscrape.com", config=config)
+
+    # Show final state
+    if STATE_FILE.exists():
+        with open(STATE_FILE, "r") as f:
+            final_state = json.load(f)
+
+        print(f"\nFinal state saved to {STATE_FILE}:")
+        print(f"  - Strategy: {final_state['strategy_type']}")
+        print(f"  - Pages crawled: {final_state['pages_crawled']}")
+        print(f"  - URLs visited: {len(final_state['visited'])}")
+        print(f"  - URLs pending: {len(final_state['pending'])}")
+
+    print(f"\nCrawled {len(results)} pages total")
+
+
+async def example_crash_and_resume():
+    """
+    Example 2: Simulate a crash and resume from checkpoint.
+
+    This demonstrates the full crash recovery workflow:
+    1. Start crawling with state persistence
+    2. "Crash" after N pages
+    3. Resume from saved state
+    4. Verify no duplicate work
+    """
+    print("\n" + "=" * 60)
+    print("Example 2: Crash and Resume")
+    print("=" * 60)
+
+    # Clean up any previous state
+    if STATE_FILE.exists():
+        STATE_FILE.unlink()
+
+    crash_after = 3
+    crawled_urls_phase1: List[str] = []
+
+    async def save_and_maybe_crash(state: Dict[str, Any]) -> None:
+        """Save state, then simulate crash after N pages."""
+        # Always save state first
+        await save_state_to_file(state)
+        crawled_urls_phase1.clear()
+        crawled_urls_phase1.extend(state["visited"])
+
+        # Simulate crash after reaching threshold
+        if state["pages_crawled"] >= crash_after:
+            raise Exception("Simulated crash! (This is intentional)")
+
+    # Phase 1: Start crawl that will "crash"
+    print(f"\n--- Phase 1: Crawl until 'crash' after {crash_after} pages ---")
+
+    strategy1 = BFSDeepCrawlStrategy(
+        max_depth=2,
+        max_pages=10,
+        on_state_change=save_and_maybe_crash,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy1,
+        verbose=False,
+    )
+
+    try:
+        async with AsyncWebCrawler(verbose=False) as crawler:
+            await crawler.arun("https://books.toscrape.com", config=config)
+    except Exception as e:
+        print(f"\n  Crash occurred: {e}")
+        print(f"  URLs crawled before crash: {len(crawled_urls_phase1)}")
+
+    # Phase 2: Resume from checkpoint
+    print("\n--- Phase 2: Resume from checkpoint ---")
+
+    saved_state = load_state_from_file()
+    if not saved_state:
+        print("  ERROR: No saved state found!")
+        return
+
+    print(f"  Loaded state: {saved_state['pages_crawled']} pages, {len(saved_state['pending'])} pending")
+
+    crawled_urls_phase2: List[str] = []
+
+    async def track_resumed_crawl(state: Dict[str, Any]) -> None:
+        """Track new URLs crawled in phase 2."""
+        await save_state_to_file(state)
+        new_urls = set(state["visited"]) - set(saved_state["visited"])
+        for url in new_urls:
+            if url not in crawled_urls_phase2:
+                crawled_urls_phase2.append(url)
+
+    strategy2 = BFSDeepCrawlStrategy(
+        max_depth=2,
+        max_pages=10,
+        resume_state=saved_state,  # Resume from checkpoint!
+        on_state_change=track_resumed_crawl,
+    )
+
+    config2 = CrawlerRunConfig(
+        deep_crawl_strategy=strategy2,
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        results = await crawler.arun("https://books.toscrape.com", config=config2)
+
+    # Verify no duplicates
+    already_crawled = set(saved_state["visited"])
+    duplicates = set(crawled_urls_phase2) & already_crawled
+
+    print(f"\n--- Results ---")
+    print(f"  Phase 1 URLs: {len(crawled_urls_phase1)}")
+    print(f"  Phase 2 new URLs: {len(crawled_urls_phase2)}")
+    print(f"  Duplicate crawls: {len(duplicates)} (should be 0)")
+    print(f"  Total results: {len(results)}")
+
+    if len(duplicates) == 0:
+        print("\n  SUCCESS: No duplicate work after resume!")
+    else:
+        print(f"\n  WARNING: Found duplicates: {duplicates}")
+
+
+async def example_export_state():
+    """
+    Example 3: Manual state export using export_state().
+
+    If you don't need real-time persistence, you can export
+    the state manually after the crawl completes.
+    """
+    print("\n" + "=" * 60)
+    print("Example 3: Manual State Export")
+    print("=" * 60)
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=1,
+        max_pages=3,
+        # No callback - state is still tracked internally
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    print("\nCrawling without callback...")
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        results = await crawler.arun("https://books.toscrape.com", config=config)
+
+    # Export state after crawl completes
+    # Note: This only works if on_state_change was set during crawl
+    # For this example, we'd need to set on_state_change to get state
+    print(f"\nCrawled {len(results)} pages")
+    print("(For manual export, set on_state_change to capture state)")
+
+
+async def example_state_structure():
+    """
+    Example 4: Understanding the state structure.
+
+    Shows the complete state dictionary that gets saved.
+    """
+    print("\n" + "=" * 60)
+    print("Example 4: State Structure")
+    print("=" * 60)
+
+    captured_state = None
+
+    async def capture_state(state: Dict[str, Any]) -> None:
+        nonlocal captured_state
+        captured_state = state
+
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=1,
+        max_pages=2,
+        on_state_change=capture_state,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=strategy,
+        verbose=False,
+    )
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        await crawler.arun("https://books.toscrape.com", config=config)
+
+    if captured_state:
+        print("\nState structure:")
+        print(json.dumps(captured_state, indent=2, default=str)[:1000] + "...")
+
+        print("\n\nKey fields:")
+        print(f"  strategy_type: '{captured_state['strategy_type']}'")
+        print(f"  visited: List of {len(captured_state['visited'])} URLs")
+        print(f"  pending: List of {len(captured_state['pending'])} queued items")
+        print(f"  depths: Dict mapping URL -> depth level")
+        print(f"  pages_crawled: {captured_state['pages_crawled']}")
+
+
+async def main():
+    """Run all examples."""
+    print("=" * 60)
+    print("Deep Crawl Crash Recovery Examples")
+    print("=" * 60)
+
+    await example_basic_state_persistence()
+    await example_crash_and_resume()
+    await example_state_structure()
+
+    # # Cleanup
+    # if STATE_FILE.exists():
+    #     STATE_FILE.unlink()
+    #     print(f"\n[Cleaned up {STATE_FILE}]")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/dfs_crawl_demo.py

@@ -0,0 +1,39 @@
+"""
+Simple demonstration of the DFS deep crawler visiting multiple pages.
+
+Run with:  python docs/examples/dfs_crawl_demo.py
+"""
+import asyncio
+
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_webcrawler import AsyncWebCrawler
+from crawl4ai.cache_context import CacheMode
+from crawl4ai.deep_crawling.dfs_strategy import DFSDeepCrawlStrategy
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+
+async def main() -> None:
+    dfs_strategy = DFSDeepCrawlStrategy(
+        max_depth=3,
+        max_pages=50,
+        include_external=False,
+    )
+
+    config = CrawlerRunConfig(
+        deep_crawl_strategy=dfs_strategy,
+        cache_mode=CacheMode.BYPASS,
+        markdown_generator=DefaultMarkdownGenerator(),
+        stream=True,
+    )
+
+    seed_url = "https://docs.python.org/3/"  # Plenty of internal links
+
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+        async for result in await crawler.arun(url=seed_url, config=config):
+            depth = result.metadata.get("depth")
+            status = "SUCCESS" if result.success else "FAILED"
+            print(f"[{status}] depth={depth} url={result.url}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/docker_client_hooks_example.py

@@ -0,0 +1,522 @@
+#!/usr/bin/env python3
+"""
+Comprehensive hooks examples using Docker Client with function objects.
+
+This approach is recommended because:
+- Write hooks as regular Python functions
+- Full IDE support (autocomplete, type checking)
+- Automatic conversion to API format
+- Reusable and testable code
+- Clean, readable syntax
+"""
+
+import asyncio
+from crawl4ai import Crawl4aiDockerClient
+
+# API_BASE_URL = "http://localhost:11235"
+API_BASE_URL = "http://localhost:11234"
+
+
+# ============================================================================
+# Hook Function Definitions
+# ============================================================================
+
+# --- All Hooks Demo ---
+async def browser_created_hook(browser, **kwargs):
+    """Called after browser is created"""
+    print("[HOOK] Browser created and ready")
+    return browser
+
+
+async def page_context_hook(page, context, **kwargs):
+    """Setup page environment"""
+    print("[HOOK] Setting up page environment")
+
+    # Set viewport
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+
+    # Add cookies
+    await context.add_cookies([{
+        "name": "test_session",
+        "value": "abc123xyz",
+        "domain": ".httpbin.org",
+        "path": "/"
+    }])
+
+    # Block resources
+    await context.route("**/*.{png,jpg,jpeg,gif}", lambda route: route.abort())
+    await context.route("**/analytics/*", lambda route: route.abort())
+
+    print("[HOOK] Environment configured")
+    return page
+
+
+async def user_agent_hook(page, context, user_agent, **kwargs):
+    """Called when user agent is updated"""
+    print(f"[HOOK] User agent: {user_agent[:50]}...")
+    return page
+
+
+async def before_goto_hook(page, context, url, **kwargs):
+    """Called before navigating to URL"""
+    print(f"[HOOK] Navigating to: {url}")
+
+    await page.set_extra_http_headers({
+        "X-Custom-Header": "crawl4ai-test",
+        "Accept-Language": "en-US"
+    })
+
+    return page
+
+
+async def after_goto_hook(page, context, url, response, **kwargs):
+    """Called after page loads"""
+    print(f"[HOOK] Page loaded: {url}")
+
+    await page.wait_for_timeout(1000)
+
+    try:
+        await page.wait_for_selector("body", timeout=2000)
+        print("[HOOK] Body element ready")
+    except:
+        print("[HOOK] Timeout, continuing")
+
+    return page
+
+
+async def execution_started_hook(page, context, **kwargs):
+    """Called when custom JS execution starts"""
+    print("[HOOK] JS execution started")
+    await page.evaluate("console.log('[HOOK] Custom JS');")
+    return page
+
+
+async def before_retrieve_hook(page, context, **kwargs):
+    """Called before retrieving HTML"""
+    print("[HOOK] Preparing HTML retrieval")
+
+    # Scroll for lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+    await page.wait_for_timeout(500)
+    await page.evaluate("window.scrollTo(0, 0);")
+
+    print("[HOOK] Scrolling complete")
+    return page
+
+
+async def before_return_hook(page, context, html, **kwargs):
+    """Called before returning HTML"""
+    print(f"[HOOK] HTML ready: {len(html)} chars")
+
+    metrics = await page.evaluate('''() => ({
+        images: document.images.length,
+        links: document.links.length,
+        scripts: document.scripts.length
+    })''')
+
+    print(f"[HOOK] Metrics - Images: {metrics['images']}, Links: {metrics['links']}")
+    return page
+
+
+# --- Authentication Hooks ---
+async def auth_context_hook(page, context, **kwargs):
+    """Setup authentication context"""
+    print("[HOOK] Setting up authentication")
+
+    # Add auth cookies
+    await context.add_cookies([{
+        "name": "auth_token",
+        "value": "fake_jwt_token",
+        "domain": ".httpbin.org",
+        "path": "/",
+        "httpOnly": True
+    }])
+
+    # Set localStorage
+    await page.evaluate('''
+        localStorage.setItem('user_id', '12345');
+        localStorage.setItem('auth_time', new Date().toISOString());
+    ''')
+
+    print("[HOOK] Auth context ready")
+    return page
+
+
+async def auth_headers_hook(page, context, url, **kwargs):
+    """Add authentication headers"""
+    print(f"[HOOK] Adding auth headers for {url}")
+
+    import base64
+    credentials = base64.b64encode(b"user:passwd").decode('ascii')
+
+    await page.set_extra_http_headers({
+        'Authorization': f'Basic {credentials}',
+        'X-API-Key': 'test-key-123'
+    })
+
+    return page
+
+
+# --- Performance Optimization Hooks ---
+async def performance_hook(page, context, **kwargs):
+    """Optimize page for performance"""
+    print("[HOOK] Optimizing for performance")
+
+    # Block resource-heavy content
+    await context.route("**/*.{png,jpg,jpeg,gif,webp,svg}", lambda r: r.abort())
+    await context.route("**/*.{woff,woff2,ttf}", lambda r: r.abort())
+    await context.route("**/*.{mp4,webm,ogg}", lambda r: r.abort())
+    await context.route("**/googletagmanager.com/*", lambda r: r.abort())
+    await context.route("**/google-analytics.com/*", lambda r: r.abort())
+    await context.route("**/facebook.com/*", lambda r: r.abort())
+
+    # Disable animations
+    await page.add_style_tag(content='''
+        *, *::before, *::after {
+            animation-duration: 0s !important;
+            transition-duration: 0s !important;
+        }
+    ''')
+
+    print("[HOOK] Optimizations applied")
+    return page
+
+
+async def cleanup_hook(page, context, **kwargs):
+    """Clean page before extraction"""
+    print("[HOOK] Cleaning page")
+
+    await page.evaluate('''() => {
+        const selectors = [
+            '.ad', '.ads', '.advertisement',
+            '.popup', '.modal', '.overlay',
+            '.cookie-banner', '.newsletter'
+        ];
+
+        selectors.forEach(sel => {
+            document.querySelectorAll(sel).forEach(el => el.remove());
+        });
+
+        document.querySelectorAll('script, style').forEach(el => el.remove());
+    }''')
+
+    print("[HOOK] Page cleaned")
+    return page
+
+
+# --- Content Extraction Hooks ---
+async def wait_dynamic_content_hook(page, context, url, response, **kwargs):
+    """Wait for dynamic content to load"""
+    print(f"[HOOK] Waiting for dynamic content on {url}")
+
+    await page.wait_for_timeout(2000)
+
+    # Click "Load More" if exists
+    try:
+        load_more = await page.query_selector('[class*="load-more"], button:has-text("Load More")')
+        if load_more:
+            await load_more.click()
+            await page.wait_for_timeout(1000)
+            print("[HOOK] Clicked 'Load More'")
+    except:
+        pass
+
+    return page
+
+
+async def extract_metadata_hook(page, context, **kwargs):
+    """Extract page metadata"""
+    print("[HOOK] Extracting metadata")
+
+    metadata = await page.evaluate('''() => {
+        const getMeta = (name) => {
+            const el = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`);
+            return el ? el.getAttribute('content') : null;
+        };
+
+        return {
+            title: document.title,
+            description: getMeta('description'),
+            author: getMeta('author'),
+            keywords: getMeta('keywords'),
+        };
+    }''')
+
+    print(f"[HOOK] Metadata: {metadata}")
+
+    # Infinite scroll
+    for i in range(3):
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+        await page.wait_for_timeout(1000)
+        print(f"[HOOK] Scroll {i+1}/3")
+
+    return page
+
+
+# --- Multi-URL Hooks ---
+async def url_specific_hook(page, context, url, **kwargs):
+    """Apply URL-specific logic"""
+    print(f"[HOOK] Processing URL: {url}")
+
+    # URL-specific headers
+    if 'html' in url:
+        await page.set_extra_http_headers({"X-Type": "HTML"})
+    elif 'json' in url:
+        await page.set_extra_http_headers({"X-Type": "JSON"})
+
+    return page
+
+
+async def track_progress_hook(page, context, url, response, **kwargs):
+    """Track crawl progress"""
+    status = response.status if response else 'unknown'
+    print(f"[HOOK] Loaded {url} - Status: {status}")
+    return page
+
+
+# ============================================================================
+# Test Functions
+# ============================================================================
+
+async def test_all_hooks_comprehensive():
+    """Test all 8 hook types"""
+    print("=" * 70)
+    print("Test 1: All Hooks Comprehensive Demo (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nCrawling with all 8 hooks...")
+
+        # Define hooks with function objects
+        hooks = {
+            "on_browser_created": browser_created_hook,
+            "on_page_context_created": page_context_hook,
+            "on_user_agent_updated": user_agent_hook,
+            "before_goto": before_goto_hook,
+            "after_goto": after_goto_hook,
+            "on_execution_started": execution_started_hook,
+            "before_retrieve_html": before_retrieve_hook,
+            "before_return_html": before_return_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks=hooks,
+            hooks_timeout=30
+        )
+
+        print("\n✅ Success!")
+        print(f"   URL: {result.url}")
+        print(f"   Success: {result.success}")
+        print(f"   HTML: {len(result.html)} chars")
+
+
+async def test_authentication_workflow():
+    """Test authentication with hooks"""
+    print("\n" + "=" * 70)
+    print("Test 2: Authentication Workflow (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting authentication...")
+
+        hooks = {
+            "on_page_context_created": auth_context_hook,
+            "before_goto": auth_headers_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/basic-auth/user/passwd"],
+            hooks=hooks,
+            hooks_timeout=15
+        )
+
+        print("\n✅ Authentication completed")
+
+        if result.success:
+            if '"authenticated"' in result.html and 'true' in result.html:
+                print("   ✅ Basic auth successful!")
+            else:
+                print("   ⚠️ Auth status unclear")
+        else:
+            print(f"   ❌ Failed: {result.error_message}")
+
+
+async def test_performance_optimization():
+    """Test performance optimization"""
+    print("\n" + "=" * 70)
+    print("Test 3: Performance Optimization (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting performance hooks...")
+
+        hooks = {
+            "on_page_context_created": performance_hook,
+            "before_retrieve_html": cleanup_hook
+        }
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            hooks=hooks,
+            hooks_timeout=10
+        )
+
+        print("\n✅ Optimization completed")
+        print(f"   HTML size: {len(result.html):,} chars")
+        print("   Resources blocked, ads removed")
+
+
+async def test_content_extraction():
+    """Test content extraction"""
+    print("\n" + "=" * 70)
+    print("Test 4: Content Extraction (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nTesting extraction hooks...")
+
+        hooks = {
+            "after_goto": wait_dynamic_content_hook,
+            "before_retrieve_html": extract_metadata_hook
+        }
+
+        result = await client.crawl(
+            ["https://www.kidocode.com/"],
+            hooks=hooks,
+            hooks_timeout=20
+        )
+
+        print("\n✅ Extraction completed")
+        print(f"   URL: {result.url}")
+        print(f"   Success: {result.success}")
+        print(f"   Metadata: {result.metadata}")
+
+
+async def test_multi_url_crawl():
+    """Test hooks with multiple URLs"""
+    print("\n" + "=" * 70)
+    print("Test 5: Multi-URL Crawl (Docker Client)")
+    print("=" * 70)
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nCrawling multiple URLs...")
+
+        hooks = {
+            "before_goto": url_specific_hook,
+            "after_goto": track_progress_hook
+        }
+
+        results = await client.crawl(
+            [
+                "https://httpbin.org/html",
+                "https://httpbin.org/json",
+                "https://httpbin.org/xml"
+            ],
+            hooks=hooks,
+            hooks_timeout=15
+        )
+
+        print("\n✅ Multi-URL crawl completed")
+        print(f"\n   Crawled {len(results)} URLs:")
+        for i, result in enumerate(results, 1):
+            status = "✅" if result.success else "❌"
+            print(f"   {status} {i}. {result.url}")
+
+
+async def test_reusable_hook_library():
+    """Test using reusable hook library"""
+    print("\n" + "=" * 70)
+    print("Test 6: Reusable Hook Library (Docker Client)")
+    print("=" * 70)
+
+    # Create a library of reusable hooks
+    class HookLibrary:
+        @staticmethod
+        async def block_images(page, context, **kwargs):
+            """Block all images"""
+            await context.route("**/*.{png,jpg,jpeg,gif}", lambda r: r.abort())
+            print("[LIBRARY] Images blocked")
+            return page
+
+        @staticmethod
+        async def block_analytics(page, context, **kwargs):
+            """Block analytics"""
+            await context.route("**/analytics/*", lambda r: r.abort())
+            await context.route("**/google-analytics.com/*", lambda r: r.abort())
+            print("[LIBRARY] Analytics blocked")
+            return page
+
+        @staticmethod
+        async def scroll_infinite(page, context, **kwargs):
+            """Handle infinite scroll"""
+            for i in range(5):
+                prev = await page.evaluate("document.body.scrollHeight")
+                await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+                await page.wait_for_timeout(1000)
+                curr = await page.evaluate("document.body.scrollHeight")
+                if curr == prev:
+                    break
+            print("[LIBRARY] Infinite scroll complete")
+            return page
+
+    async with Crawl4aiDockerClient(base_url=API_BASE_URL, verbose=False) as client:
+        print("\nUsing hook library...")
+
+        hooks = {
+            "on_page_context_created": HookLibrary.block_images,
+            "before_retrieve_html": HookLibrary.scroll_infinite
+        }
+
+        result = await client.crawl(
+            ["https://www.kidocode.com/"],
+            hooks=hooks,
+            hooks_timeout=20
+        )
+
+        print("\n✅ Library hooks completed")
+        print(f"   Success: {result.success}")
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+async def main():
+    """Run all Docker client hook examples"""
+    print("🔧 Crawl4AI Docker Client - Hooks Examples (Function-Based)")
+    print("Using Python function objects with automatic conversion")
+    print("=" * 70)
+
+    tests = [
+        ("All Hooks Demo", test_all_hooks_comprehensive),
+        ("Authentication", test_authentication_workflow),
+        ("Performance", test_performance_optimization),
+        ("Extraction", test_content_extraction),
+        ("Multi-URL", test_multi_url_crawl),
+        ("Hook Library", test_reusable_hook_library)
+    ]
+
+    for i, (name, test_func) in enumerate(tests, 1):
+        try:
+            await test_func()
+            print(f"\n✅ Test {i}/{len(tests)}: {name} completed\n")
+        except Exception as e:
+            print(f"\n❌ Test {i}/{len(tests)}: {name} failed: {e}\n")
+            import traceback
+            traceback.print_exc()
+
+    print("=" * 70)
+    print("🎉 All Docker client hook examples completed!")
+    print("\n💡 Key Benefits of Function-Based Hooks:")
+    print("   • Write as regular Python functions")
+    print("   • Full IDE support (autocomplete, types)")
+    print("   • Automatic conversion to API format")
+    print("   • Reusable across projects")
+    print("   • Clean, readable code")
+    print("   • Easy to test and debug")
+    print("=" * 70)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/docker_webhook_example.py

@@ -0,0 +1,461 @@
+"""
+Docker Webhook Example for Crawl4AI
+
+This example demonstrates how to use webhooks with the Crawl4AI job queue API.
+Instead of polling for results, webhooks notify your application when jobs complete.
+
+Supports both:
+- /crawl/job - Raw crawling with markdown extraction
+- /llm/job - LLM-powered content extraction
+
+Prerequisites:
+1. Crawl4AI Docker container running on localhost:11235
+2. Flask installed: pip install flask requests
+3. LLM API key configured in .llm.env (for LLM extraction examples)
+
+Usage:
+1. Run this script: python docker_webhook_example.py
+2. The webhook server will start on http://localhost:8080
+3. Jobs will be submitted and webhooks will be received automatically
+"""
+
+import requests
+import json
+import time
+from flask import Flask, request, jsonify
+from threading import Thread
+
+# Configuration
+CRAWL4AI_BASE_URL = "http://localhost:11235"
+WEBHOOK_BASE_URL = "http://localhost:8080"  # Your webhook receiver URL
+
+# Initialize Flask app for webhook receiver
+app = Flask(__name__)
+
+# Store received webhook data for demonstration
+received_webhooks = []
+
+
+@app.route('/webhooks/crawl-complete', methods=['POST'])
+def handle_crawl_webhook():
+    """
+    Webhook handler that receives notifications when crawl jobs complete.
+
+    Payload structure:
+    {
+        "task_id": "crawl_abc123",
+        "task_type": "crawl",
+        "status": "completed" or "failed",
+        "timestamp": "2025-10-21T10:30:00.000000+00:00",
+        "urls": ["https://example.com"],
+        "error": "error message" (only if failed),
+        "data": {...} (only if webhook_data_in_payload=True)
+    }
+    """
+    payload = request.json
+    print(f"\n{'='*60}")
+    print(f"📬 Webhook received for task: {payload['task_id']}")
+    print(f"   Status: {payload['status']}")
+    print(f"   Timestamp: {payload['timestamp']}")
+    print(f"   URLs: {payload['urls']}")
+
+    if payload['status'] == 'completed':
+        # If data is in payload, process it directly
+        if 'data' in payload:
+            print(f"   ✅ Data included in webhook")
+            data = payload['data']
+            # Process the crawl results here
+            for result in data.get('results', []):
+                print(f"      - Crawled: {result.get('url')}")
+                print(f"      - Markdown length: {len(result.get('markdown', ''))}")
+        else:
+            # Fetch results from API if not included
+            print(f"   📥 Fetching results from API...")
+            task_id = payload['task_id']
+            result_response = requests.get(f"{CRAWL4AI_BASE_URL}/crawl/job/{task_id}")
+            if result_response.ok:
+                data = result_response.json()
+                print(f"   ✅ Results fetched successfully")
+                # Process the crawl results here
+                for result in data['result'].get('results', []):
+                    print(f"      - Crawled: {result.get('url')}")
+                    print(f"      - Markdown length: {len(result.get('markdown', ''))}")
+
+    elif payload['status'] == 'failed':
+        print(f"   ❌ Job failed: {payload.get('error', 'Unknown error')}")
+
+    print(f"{'='*60}\n")
+
+    # Store webhook for demonstration
+    received_webhooks.append(payload)
+
+    # Return 200 OK to acknowledge receipt
+    return jsonify({"status": "received"}), 200
+
+
+@app.route('/webhooks/llm-complete', methods=['POST'])
+def handle_llm_webhook():
+    """
+    Webhook handler that receives notifications when LLM extraction jobs complete.
+
+    Payload structure:
+    {
+        "task_id": "llm_1698765432_12345",
+        "task_type": "llm_extraction",
+        "status": "completed" or "failed",
+        "timestamp": "2025-10-21T10:30:00.000000+00:00",
+        "urls": ["https://example.com/article"],
+        "error": "error message" (only if failed),
+        "data": {"extracted_content": {...}} (only if webhook_data_in_payload=True)
+    }
+    """
+    payload = request.json
+    print(f"\n{'='*60}")
+    print(f"🤖 LLM Webhook received for task: {payload['task_id']}")
+    print(f"   Task Type: {payload['task_type']}")
+    print(f"   Status: {payload['status']}")
+    print(f"   Timestamp: {payload['timestamp']}")
+    print(f"   URL: {payload['urls'][0]}")
+
+    if payload['status'] == 'completed':
+        # If data is in payload, process it directly
+        if 'data' in payload:
+            print(f"   ✅ Data included in webhook")
+            data = payload['data']
+            # Webhook wraps extracted content in 'extracted_content' field
+            extracted = data.get('extracted_content', {})
+            print(f"      - Extracted content:")
+            print(f"        {json.dumps(extracted, indent=8)}")
+        else:
+            # Fetch results from API if not included
+            print(f"   📥 Fetching results from API...")
+            task_id = payload['task_id']
+            result_response = requests.get(f"{CRAWL4AI_BASE_URL}/llm/job/{task_id}")
+            if result_response.ok:
+                data = result_response.json()
+                print(f"   ✅ Results fetched successfully")
+                # API returns unwrapped content in 'result' field
+                extracted = data['result']
+                print(f"      - Extracted content:")
+                print(f"        {json.dumps(extracted, indent=8)}")
+
+    elif payload['status'] == 'failed':
+        print(f"   ❌ Job failed: {payload.get('error', 'Unknown error')}")
+
+    print(f"{'='*60}\n")
+
+    # Store webhook for demonstration
+    received_webhooks.append(payload)
+
+    # Return 200 OK to acknowledge receipt
+    return jsonify({"status": "received"}), 200
+
+
+def start_webhook_server():
+    """Start the Flask webhook server in a separate thread"""
+    app.run(host='0.0.0.0', port=8080, debug=False, use_reloader=False)
+
+
+def submit_crawl_job_with_webhook(urls, webhook_url, include_data=False):
+    """
+    Submit a crawl job with webhook notification.
+
+    Args:
+        urls: List of URLs to crawl
+        webhook_url: URL to receive webhook notifications
+        include_data: Whether to include full results in webhook payload
+
+    Returns:
+        task_id: The job's task identifier
+    """
+    payload = {
+        "urls": urls,
+        "browser_config": {"headless": True},
+        "crawler_config": {"cache_mode": "bypass"},
+        "webhook_config": {
+            "webhook_url": webhook_url,
+            "webhook_data_in_payload": include_data,
+            # Optional: Add custom headers for authentication
+            # "webhook_headers": {
+            #     "X-Webhook-Secret": "your-secret-token"
+            # }
+        }
+    }
+
+    print(f"\n🚀 Submitting crawl job...")
+    print(f"   URLs: {urls}")
+    print(f"   Webhook: {webhook_url}")
+    print(f"   Include data: {include_data}")
+
+    response = requests.post(
+        f"{CRAWL4AI_BASE_URL}/crawl/job",
+        json=payload,
+        headers={"Content-Type": "application/json"}
+    )
+
+    if response.ok:
+        data = response.json()
+        task_id = data['task_id']
+        print(f"   ✅ Job submitted successfully")
+        print(f"   Task ID: {task_id}")
+        return task_id
+    else:
+        print(f"   ❌ Failed to submit job: {response.text}")
+        return None
+
+
+def submit_llm_job_with_webhook(url, query, webhook_url, include_data=False, schema=None, provider=None):
+    """
+    Submit an LLM extraction job with webhook notification.
+
+    Args:
+        url: URL to extract content from
+        query: Instruction for the LLM (e.g., "Extract article title and author")
+        webhook_url: URL to receive webhook notifications
+        include_data: Whether to include full results in webhook payload
+        schema: Optional JSON schema for structured extraction
+        provider: Optional LLM provider (e.g., "openai/gpt-4o-mini")
+
+    Returns:
+        task_id: The job's task identifier
+    """
+    payload = {
+        "url": url,
+        "q": query,
+        "cache": False,
+        "webhook_config": {
+            "webhook_url": webhook_url,
+            "webhook_data_in_payload": include_data,
+            # Optional: Add custom headers for authentication
+            # "webhook_headers": {
+            #     "X-Webhook-Secret": "your-secret-token"
+            # }
+        }
+    }
+
+    if schema:
+        payload["schema"] = schema
+
+    if provider:
+        payload["provider"] = provider
+
+    print(f"\n🤖 Submitting LLM extraction job...")
+    print(f"   URL: {url}")
+    print(f"   Query: {query}")
+    print(f"   Webhook: {webhook_url}")
+    print(f"   Include data: {include_data}")
+    if provider:
+        print(f"   Provider: {provider}")
+
+    response = requests.post(
+        f"{CRAWL4AI_BASE_URL}/llm/job",
+        json=payload,
+        headers={"Content-Type": "application/json"}
+    )
+
+    if response.ok:
+        data = response.json()
+        task_id = data['task_id']
+        print(f"   ✅ Job submitted successfully")
+        print(f"   Task ID: {task_id}")
+        return task_id
+    else:
+        print(f"   ❌ Failed to submit job: {response.text}")
+        return None
+
+
+def submit_job_without_webhook(urls):
+    """
+    Submit a job without webhook (traditional polling approach).
+
+    Args:
+        urls: List of URLs to crawl
+
+    Returns:
+        task_id: The job's task identifier
+    """
+    payload = {
+        "urls": urls,
+        "browser_config": {"headless": True},
+        "crawler_config": {"cache_mode": "bypass"}
+    }
+
+    print(f"\n🚀 Submitting crawl job (without webhook)...")
+    print(f"   URLs: {urls}")
+
+    response = requests.post(
+        f"{CRAWL4AI_BASE_URL}/crawl/job",
+        json=payload
+    )
+
+    if response.ok:
+        data = response.json()
+        task_id = data['task_id']
+        print(f"   ✅ Job submitted successfully")
+        print(f"   Task ID: {task_id}")
+        return task_id
+    else:
+        print(f"   ❌ Failed to submit job: {response.text}")
+        return None
+
+
+def poll_job_status(task_id, timeout=60):
+    """
+    Poll for job status (used when webhook is not configured).
+
+    Args:
+        task_id: The job's task identifier
+        timeout: Maximum time to wait in seconds
+    """
+    print(f"\n⏳ Polling for job status...")
+    start_time = time.time()
+
+    while time.time() - start_time < timeout:
+        response = requests.get(f"{CRAWL4AI_BASE_URL}/crawl/job/{task_id}")
+
+        if response.ok:
+            data = response.json()
+            status = data.get('status', 'unknown')
+
+            if status == 'completed':
+                print(f"   ✅ Job completed!")
+                return data
+            elif status == 'failed':
+                print(f"   ❌ Job failed: {data.get('error', 'Unknown error')}")
+                return data
+            else:
+                print(f"   ⏳ Status: {status}, waiting...")
+                time.sleep(2)
+        else:
+            print(f"   ❌ Failed to get status: {response.text}")
+            return None
+
+    print(f"   ⏰ Timeout reached")
+    return None
+
+
+def main():
+    """Run the webhook demonstration"""
+
+    # Check if Crawl4AI is running
+    try:
+        health = requests.get(f"{CRAWL4AI_BASE_URL}/health", timeout=5)
+        print(f"✅ Crawl4AI is running: {health.json()}")
+    except:
+        print(f"❌ Cannot connect to Crawl4AI at {CRAWL4AI_BASE_URL}")
+        print("   Please make sure Docker container is running:")
+        print("   docker run -d -p 11235:11235 --name crawl4ai unclecode/crawl4ai:latest")
+        return
+
+    # Start webhook server in background thread
+    print(f"\n🌐 Starting webhook server at {WEBHOOK_BASE_URL}...")
+    webhook_thread = Thread(target=start_webhook_server, daemon=True)
+    webhook_thread.start()
+    time.sleep(2)  # Give server time to start
+
+    # Example 1: Job with webhook (notification only, fetch data separately)
+    print(f"\n{'='*60}")
+    print("Example 1: Webhook Notification Only")
+    print(f"{'='*60}")
+    task_id_1 = submit_crawl_job_with_webhook(
+        urls=["https://example.com"],
+        webhook_url=f"{WEBHOOK_BASE_URL}/webhooks/crawl-complete",
+        include_data=False
+    )
+
+    # Example 2: Job with webhook (data included in payload)
+    time.sleep(5)  # Wait a bit between requests
+    print(f"\n{'='*60}")
+    print("Example 2: Webhook with Full Data")
+    print(f"{'='*60}")
+    task_id_2 = submit_crawl_job_with_webhook(
+        urls=["https://www.python.org"],
+        webhook_url=f"{WEBHOOK_BASE_URL}/webhooks/crawl-complete",
+        include_data=True
+    )
+
+    # Example 3: LLM extraction with webhook (notification only)
+    time.sleep(5)  # Wait a bit between requests
+    print(f"\n{'='*60}")
+    print("Example 3: LLM Extraction with Webhook (Notification Only)")
+    print(f"{'='*60}")
+    task_id_3 = submit_llm_job_with_webhook(
+        url="https://www.example.com",
+        query="Extract the main heading and description from this page.",
+        webhook_url=f"{WEBHOOK_BASE_URL}/webhooks/llm-complete",
+        include_data=False,
+        provider="openai/gpt-4o-mini"
+    )
+
+    # Example 4: LLM extraction with webhook (data included + schema)
+    time.sleep(5)  # Wait a bit between requests
+    print(f"\n{'='*60}")
+    print("Example 4: LLM Extraction with Schema and Full Data")
+    print(f"{'='*60}")
+
+    # Define a schema for structured extraction
+    schema = json.dumps({
+        "type": "object",
+        "properties": {
+            "title": {"type": "string", "description": "Page title"},
+            "description": {"type": "string", "description": "Page description"}
+        },
+        "required": ["title"]
+    })
+
+    task_id_4 = submit_llm_job_with_webhook(
+        url="https://www.python.org",
+        query="Extract the title and description of this website",
+        webhook_url=f"{WEBHOOK_BASE_URL}/webhooks/llm-complete",
+        include_data=True,
+        schema=schema,
+        provider="openai/gpt-4o-mini"
+    )
+
+    # Example 5: Traditional polling (no webhook)
+    time.sleep(5)  # Wait a bit between requests
+    print(f"\n{'='*60}")
+    print("Example 5: Traditional Polling (No Webhook)")
+    print(f"{'='*60}")
+    task_id_5 = submit_job_without_webhook(
+        urls=["https://github.com"]
+    )
+    if task_id_5:
+        result = poll_job_status(task_id_5)
+        if result and result.get('status') == 'completed':
+            print(f"   ✅ Results retrieved via polling")
+
+    # Wait for webhooks to arrive
+    print(f"\n⏳ Waiting for webhooks to be received...")
+    time.sleep(30)  # Give jobs time to complete and webhooks to arrive (longer for LLM)
+
+    # Summary
+    print(f"\n{'='*60}")
+    print("Summary")
+    print(f"{'='*60}")
+    print(f"Total webhooks received: {len(received_webhooks)}")
+
+    crawl_webhooks = [w for w in received_webhooks if w['task_type'] == 'crawl']
+    llm_webhooks = [w for w in received_webhooks if w['task_type'] == 'llm_extraction']
+
+    print(f"\n📊 Breakdown:")
+    print(f"   - Crawl webhooks: {len(crawl_webhooks)}")
+    print(f"   - LLM extraction webhooks: {len(llm_webhooks)}")
+
+    print(f"\n📋 Details:")
+    for i, webhook in enumerate(received_webhooks, 1):
+        task_type = webhook['task_type']
+        icon = "🕷️" if task_type == "crawl" else "🤖"
+        print(f"{i}. {icon} Task {webhook['task_id']}: {webhook['status']} ({task_type})")
+
+    print(f"\n✅ Demo completed!")
+    print(f"\n💡 Pro tips:")
+    print(f"   - In production, your webhook URL should be publicly accessible")
+    print(f"     (e.g., https://myapp.com/webhooks) or use ngrok for testing")
+    print(f"   - Both /crawl/job and /llm/job support the same webhook configuration")
+    print(f"   - Use webhook_data_in_payload=true to get results directly in the webhook")
+    print(f"   - LLM jobs may take longer, adjust timeouts accordingly")
+
+
+if __name__ == "__main__":
+    main()

docs/examples/nst_proxy/api_proxy_example.py

@@ -0,0 +1,48 @@
+"""
+NSTProxy Integration Examples for crawl4ai
+------------------------------------------
+
+NSTProxy is a premium residential proxy provider.
+👉 Purchase Proxies: https://nstproxy.com
+💰 Use coupon code "crawl4ai" for 10% off your plan.
+
+"""
+import asyncio, requests
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+
+async def main():
+    """
+    Example: Dynamically fetch a proxy from NSTProxy API before crawling.
+    """
+    NST_TOKEN = "YOUR_NST_PROXY_TOKEN"  # Get from https://app.nstproxy.com/profile
+    CHANNEL_ID = "YOUR_NST_PROXY_CHANNEL_ID"  # Your NSTProxy Channel ID
+    country = "ANY"  # e.g. "ANY", "US", "DE"
+
+    # Fetch proxy from NSTProxy API
+    api_url = (
+        f"https://api.nstproxy.com/api/v1/generate/apiproxies"
+        f"?fType=2&channelId={CHANNEL_ID}&country={country}"
+        f"&protocol=http&sessionDuration=10&count=1&token={NST_TOKEN}"
+    )
+    response = requests.get(api_url, timeout=10).json()
+    proxy = response[0]
+
+    ip = proxy.get("ip")
+    port = proxy.get("port")
+    username = proxy.get("username", "")
+    password = proxy.get("password", "")
+
+    browser_config = BrowserConfig(proxy_config={
+        "server": f"http://{ip}:{port}",
+        "username": username,
+        "password": password,
+    })
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[API Proxy] Status:", result.status_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/nst_proxy/auth_proxy_example.py

@@ -0,0 +1,31 @@
+"""
+NSTProxy Integration Examples for crawl4ai
+------------------------------------------
+
+NSTProxy is a premium residential proxy provider.
+👉 Purchase Proxies: https://nstproxy.com
+💰 Use coupon code "crawl4ai" for 10% off your plan.
+
+"""
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+
+async def main():
+    """
+    Example: Use NSTProxy with manual username/password authentication.
+    """
+
+    browser_config = BrowserConfig(proxy_config={
+        "server": "http://gate.nstproxy.io:24125",
+        "username": "your_username",
+        "password": "your_password",
+    })
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[Auth Proxy] Status:", result.status_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/nst_proxy/basic_proxy_example.py

@@ -0,0 +1,29 @@
+"""
+NSTProxy Integration Examples for crawl4ai
+------------------------------------------
+
+NSTProxy is a premium residential proxy provider.
+👉 Purchase Proxies: https://nstproxy.com
+💰 Use coupon code "crawl4ai" for 10% off your plan.
+
+"""
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+
+async def main():
+    # Using HTTP proxy
+    browser_config = BrowserConfig(proxy_config={"server": "http://gate.nstproxy.io:24125"})
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[HTTP Proxy] Status:", result.status_code)
+
+    # Using SOCKS proxy
+    browser_config = BrowserConfig(proxy_config={"server": "socks5://gate.nstproxy.io:24125"})
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[SOCKS5 Proxy] Status:", result.status_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/nst_proxy/nstproxy_example.py

@@ -0,0 +1,39 @@
+"""
+NSTProxy Integration Examples for crawl4ai
+------------------------------------------
+
+NSTProxy is a premium residential proxy provider.
+👉 Purchase Proxies: https://nstproxy.com
+💰 Use coupon code "crawl4ai" for 10% off your plan.
+
+"""
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+
+async def main():
+    """
+    Example: Using NSTProxy with AsyncWebCrawler.
+    """
+
+    NST_TOKEN = "YOUR_NST_PROXY_TOKEN"  # Get from https://app.nstproxy.com/profile
+    CHANNEL_ID = "YOUR_NST_PROXY_CHANNEL_ID"  # Your NSTProxy Channel ID
+
+    browser_config = BrowserConfig()
+    browser_config.set_nstproxy(
+        token=NST_TOKEN,
+        channel_id=CHANNEL_ID,
+        country="ANY",  # e.g. "US", "JP", or "ANY"
+        state="",  # optional, leave empty if not needed
+        city="",  # optional, leave empty if not needed
+        session_duration=0  # Session duration in minutes,0 = rotate on every request
+    )
+
+    # === Run crawler ===
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com")
+        print("[Nstproxy] Status:", result.status_code)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/examples/prefetch_two_phase_crawl.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+Prefetch Mode and Two-Phase Crawling Example
+
+Prefetch mode is a fast path that skips heavy processing and returns
+only HTML + links. This is ideal for:
+
+- Site mapping: Quickly discover all URLs
+- Selective crawling: Find URLs first, then process only what you need
+- Link validation: Check which pages exist without full processing
+- Crawl planning: Estimate size before committing resources
+
+Key concept:
+- `prefetch=True` in CrawlerRunConfig enables fast link-only extraction
+- Skips: markdown generation, content scraping, media extraction, LLM extraction
+- Returns: HTML and links dictionary
+
+Performance benefit: ~5-10x faster than full processing
+"""
+
+import asyncio
+import time
+from typing import List, Dict
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+
+async def example_basic_prefetch():
+    """
+    Example 1: Basic prefetch mode.
+
+    Shows how prefetch returns HTML and links without heavy processing.
+    """
+    print("\n" + "=" * 60)
+    print("Example 1: Basic Prefetch Mode")
+    print("=" * 60)
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # Enable prefetch mode
+        config = CrawlerRunConfig(prefetch=True)
+
+        print("\nFetching with prefetch=True...")
+        result = await crawler.arun("https://books.toscrape.com", config=config)
+
+        print(f"\nResult summary:")
+        print(f"  Success: {result.success}")
+        print(f"  HTML length: {len(result.html) if result.html else 0} chars")
+        print(f"  Internal links: {len(result.links.get('internal', []))}")
+        print(f"  External links: {len(result.links.get('external', []))}")
+
+        # These should be None/empty in prefetch mode
+        print(f"\n  Skipped processing:")
+        print(f"    Markdown: {result.markdown}")
+        print(f"    Cleaned HTML: {result.cleaned_html}")
+        print(f"    Extracted content: {result.extracted_content}")
+
+        # Show some discovered links
+        internal_links = result.links.get("internal", [])
+        if internal_links:
+            print(f"\n  Sample internal links:")
+            for link in internal_links[:5]:
+                print(f"    - {link['href'][:60]}...")
+
+
+async def example_performance_comparison():
+    """
+    Example 2: Compare prefetch vs full processing performance.
+    """
+    print("\n" + "=" * 60)
+    print("Example 2: Performance Comparison")
+    print("=" * 60)
+
+    url = "https://books.toscrape.com"
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # Warm up - first request is slower due to browser startup
+        await crawler.arun(url, config=CrawlerRunConfig())
+
+        # Prefetch mode timing
+        start = time.time()
+        prefetch_result = await crawler.arun(url, config=CrawlerRunConfig(prefetch=True))
+        prefetch_time = time.time() - start
+
+        # Full processing timing
+        start = time.time()
+        full_result = await crawler.arun(url, config=CrawlerRunConfig())
+        full_time = time.time() - start
+
+        print(f"\nTiming comparison:")
+        print(f"  Prefetch mode: {prefetch_time:.3f}s")
+        print(f"  Full processing: {full_time:.3f}s")
+        print(f"  Speedup: {full_time / prefetch_time:.1f}x faster")
+
+        print(f"\nOutput comparison:")
+        print(f"  Prefetch - Links found: {len(prefetch_result.links.get('internal', []))}")
+        print(f"  Full - Links found: {len(full_result.links.get('internal', []))}")
+        print(f"  Full - Markdown length: {len(full_result.markdown.raw_markdown) if full_result.markdown else 0}")
+
+
+async def example_two_phase_crawl():
+    """
+    Example 3: Two-phase crawling pattern.
+
+    Phase 1: Fast discovery with prefetch
+    Phase 2: Full processing on selected URLs
+    """
+    print("\n" + "=" * 60)
+    print("Example 3: Two-Phase Crawling")
+    print("=" * 60)
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # ═══════════════════════════════════════════════════════════
+        # Phase 1: Fast URL discovery
+        # ═══════════════════════════════════════════════════════════
+        print("\n--- Phase 1: Fast Discovery ---")
+
+        prefetch_config = CrawlerRunConfig(prefetch=True)
+        start = time.time()
+        discovery = await crawler.arun("https://books.toscrape.com", config=prefetch_config)
+        discovery_time = time.time() - start
+
+        all_urls = [link["href"] for link in discovery.links.get("internal", [])]
+        print(f"  Discovered {len(all_urls)} URLs in {discovery_time:.2f}s")
+
+        # Filter to URLs we care about (e.g., book detail pages)
+        # On books.toscrape.com, book pages contain "catalogue/" but not "category/"
+        book_urls = [
+            url for url in all_urls
+            if "catalogue/" in url and "category/" not in url
+        ][:5]  # Limit to 5 for demo
+
+        print(f"  Filtered to {len(book_urls)} book pages")
+
+        # ═══════════════════════════════════════════════════════════
+        # Phase 2: Full processing on selected URLs
+        # ═══════════════════════════════════════════════════════════
+        print("\n--- Phase 2: Full Processing ---")
+
+        full_config = CrawlerRunConfig(
+            word_count_threshold=10,
+            remove_overlay_elements=True,
+        )
+
+        results = []
+        start = time.time()
+
+        for url in book_urls:
+            result = await crawler.arun(url, config=full_config)
+            if result.success:
+                results.append(result)
+                title = result.url.split("/")[-2].replace("-", " ").title()[:40]
+                md_len = len(result.markdown.raw_markdown) if result.markdown else 0
+                print(f"    Processed: {title}... ({md_len} chars)")
+
+        processing_time = time.time() - start
+        print(f"\n  Processed {len(results)} pages in {processing_time:.2f}s")
+
+        # ═══════════════════════════════════════════════════════════
+        # Summary
+        # ═══════════════════════════════════════════════════════════
+        print(f"\n--- Summary ---")
+        print(f"  Discovery phase: {discovery_time:.2f}s ({len(all_urls)} URLs)")
+        print(f"  Processing phase: {processing_time:.2f}s ({len(results)} pages)")
+        print(f"  Total time: {discovery_time + processing_time:.2f}s")
+        print(f"  URLs skipped: {len(all_urls) - len(book_urls)} (not matching filter)")
+
+
+async def example_prefetch_with_deep_crawl():
+    """
+    Example 4: Combine prefetch with deep crawl strategy.
+
+    Use prefetch mode during deep crawl for maximum speed.
+    """
+    print("\n" + "=" * 60)
+    print("Example 4: Prefetch with Deep Crawl")
+    print("=" * 60)
+
+    from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        # Deep crawl with prefetch - maximum discovery speed
+        config = CrawlerRunConfig(
+            prefetch=True,  # Fast mode
+            deep_crawl_strategy=BFSDeepCrawlStrategy(
+                max_depth=1,
+                max_pages=10,
+            )
+        )
+
+        print("\nDeep crawling with prefetch mode...")
+        start = time.time()
+
+        result_container = await crawler.arun("https://books.toscrape.com", config=config)
+
+        # Handle iterator result from deep crawl
+        if hasattr(result_container, '__iter__'):
+            results = list(result_container)
+        else:
+            results = [result_container]
+
+        elapsed = time.time() - start
+
+        # Collect all discovered links
+        all_internal_links = set()
+        all_external_links = set()
+
+        for result in results:
+            for link in result.links.get("internal", []):
+                all_internal_links.add(link["href"])
+            for link in result.links.get("external", []):
+                all_external_links.add(link["href"])
+
+        print(f"\nResults:")
+        print(f"  Pages crawled: {len(results)}")
+        print(f"  Total internal links discovered: {len(all_internal_links)}")
+        print(f"  Total external links discovered: {len(all_external_links)}")
+        print(f"  Time: {elapsed:.2f}s")
+
+
+async def example_prefetch_with_raw_html():
+    """
+    Example 5: Prefetch with raw HTML input.
+
+    You can also use prefetch mode with raw: URLs for cached content.
+    """
+    print("\n" + "=" * 60)
+    print("Example 5: Prefetch with Raw HTML")
+    print("=" * 60)
+
+    sample_html = """
+    <html>
+        <head><title>Sample Page</title></head>
+        <body>
+            <h1>Hello World</h1>
+            <nav>
+                <a href="/page1">Internal Page 1</a>
+                <a href="/page2">Internal Page 2</a>
+                <a href="https://example.com/external">External Link</a>
+            </nav>
+            <main>
+                <p>This is the main content with <a href="/page3">another link</a>.</p>
+            </main>
+        </body>
+    </html>
+    """
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        config = CrawlerRunConfig(
+            prefetch=True,
+            base_url="https://mysite.com",  # For resolving relative links
+        )
+
+        result = await crawler.arun(f"raw:{sample_html}", config=config)
+
+        print(f"\nExtracted from raw HTML:")
+        print(f"  Internal links: {len(result.links.get('internal', []))}")
+        for link in result.links.get("internal", []):
+            print(f"    - {link['href']} ({link['text']})")
+
+        print(f"\n  External links: {len(result.links.get('external', []))}")
+        for link in result.links.get("external", []):
+            print(f"    - {link['href']} ({link['text']})")
+
+
+async def main():
+    """Run all examples."""
+    print("=" * 60)
+    print("Prefetch Mode and Two-Phase Crawling Examples")
+    print("=" * 60)
+
+    await example_basic_prefetch()
+    await example_performance_comparison()
+    await example_two_phase_crawl()
+    await example_prefetch_with_deep_crawl()
+    await example_prefetch_with_raw_html()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/md_v2/advanced/identity-based-crawling.md

@@ -82,6 +82,42 @@ If you installed Crawl4AI (which installs Playwright under the hood), you alread
 
 ---
 
+### Creating a Profile Using the Crawl4AI CLI (Easiest)
+
+If you prefer a guided, interactive setup, use the built-in CLI to create and manage persistent browser profiles.
+
+1.⠀Launch the profile manager:
+   ```bash
+   crwl profiles
+   ```
+
+2.⠀Choose "Create new profile" and enter a profile name. A Chromium window opens so you can log in to sites and configure settings. When finished, return to the terminal and press `q` to save the profile.
+
+3.⠀Profiles are saved under `~/.crawl4ai/profiles/<profile_name>` (for example: `/home/<you>/.crawl4ai/profiles/test_profile_1`) along with a `storage_state.json` for cookies and session data.
+
+4.⠀Optionally, choose "List profiles" in the CLI to view available profiles and their paths.
+
+5.⠀Use the saved path with `BrowserConfig.user_data_dir`:
+   ```python
+   from crawl4ai import AsyncWebCrawler, BrowserConfig
+
+   profile_path = "/home/<you>/.crawl4ai/profiles/test_profile_1"
+
+   browser_config = BrowserConfig(
+       headless=True,
+       use_managed_browser=True,
+       user_data_dir=profile_path,
+       browser_type="chromium",
+   )
+
+   async with AsyncWebCrawler(config=browser_config) as crawler:
+       result = await crawler.arun(url="https://example.com/private")
+   ```
+
+The CLI also supports listing and deleting profiles, and even testing a crawl directly from the menu.
+
+---
+
 ## 3. Using Managed Browsers in Crawl4AI
 
 Once you have a data directory with your session data, pass it to **`BrowserConfig`**:

docs/md_v2/advanced/proxy-security.md

@@ -1,98 +1,304 @@
-# Proxy 
+# Proxy & Security
+
+This guide covers proxy configuration and security features in Crawl4AI, including SSL certificate analysis and proxy rotation strategies.
+
+## Understanding Proxy Configuration
+
+Crawl4AI recommends configuring proxies per request through `CrawlerRunConfig.proxy_config`. This gives you precise control, enables rotation strategies, and keeps examples simple enough to copy, paste, and run.
 
 ## Basic Proxy Setup
 
-Simple proxy configuration with `BrowserConfig`:
+Configure proxies that apply to each crawl operation:
 
 ```python
-from crawl4ai.async_configs import BrowserConfig
-
-# Using HTTP proxy
-browser_config = BrowserConfig(proxy_config={"server": "http://proxy.example.com:8080"})
-async with AsyncWebCrawler(config=browser_config) as crawler:
-    result = await crawler.arun(url="https://example.com")
-
-# Using SOCKS proxy
-browser_config = BrowserConfig(proxy_config={"server": "socks5://proxy.example.com:1080"})
-async with AsyncWebCrawler(config=browser_config) as crawler:
-    result = await crawler.arun(url="https://example.com")
-```
-
-## Authenticated Proxy
-
-Use an authenticated proxy with `BrowserConfig`:
-
-```python
-from crawl4ai.async_configs import BrowserConfig
-
-browser_config = BrowserConfig(proxy_config={
-    "server": "http://[host]:[port]",
-    "username": "[username]",
-    "password": "[password]",
-})
-async with AsyncWebCrawler(config=browser_config) as crawler:
-    result = await crawler.arun(url="https://example.com")
-```
-
-
-## Rotating Proxies 
-
-Example using a proxy rotation service dynamically:
-
-```python
-import re
-from crawl4ai import (
-    AsyncWebCrawler,
-    BrowserConfig,
-    CrawlerRunConfig,
-    CacheMode,
-    RoundRobinProxyStrategy,
-)
 import asyncio
-from crawl4ai import ProxyConfig
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, ProxyConfig
+
+run_config = CrawlerRunConfig(proxy_config=ProxyConfig(server="http://proxy.example.com:8080"))
+# run_config = CrawlerRunConfig(proxy_config={"server": "http://proxy.example.com:8080"})
+# run_config = CrawlerRunConfig(proxy_config="http://proxy.example.com:8080")
+
+
 async def main():
-    # Load proxies and create rotation strategy
+    browser_config = BrowserConfig()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com", config=run_config)
+        print(f"Success: {result.success} -> {result.url}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+!!! note "Why request-level?"
+    `CrawlerRunConfig.proxy_config` keeps each request self-contained, so swapping proxies or rotation strategies is just a matter of building a new run configuration.
+
+## Supported Proxy Formats
+
+The `ProxyConfig.from_string()` method supports multiple formats:
+
+```python
+from crawl4ai import ProxyConfig
+
+# HTTP proxy with authentication
+proxy1 = ProxyConfig.from_string("http://user:pass@192.168.1.1:8080")
+
+# HTTPS proxy
+proxy2 = ProxyConfig.from_string("https://proxy.example.com:8080")
+
+# SOCKS5 proxy
+proxy3 = ProxyConfig.from_string("socks5://proxy.example.com:1080")
+
+# Simple IP:port format
+proxy4 = ProxyConfig.from_string("192.168.1.1:8080")
+
+# IP:port:user:pass format
+proxy5 = ProxyConfig.from_string("192.168.1.1:8080:user:pass")
+```
+
+## Authenticated Proxies
+
+For proxies requiring authentication:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler,BrowserConfig, CrawlerRunConfig, ProxyConfig
+
+run_config = CrawlerRunConfig(
+    proxy_config=ProxyConfig(
+        server="http://proxy.example.com:8080",
+        username="your_username",
+        password="your_password",
+    )
+)
+# Or dictionary style:
+# run_config = CrawlerRunConfig(proxy_config={
+#     "server": "http://proxy.example.com:8080",
+#     "username": "your_username",
+#     "password": "your_password",
+# })
+
+
+async def main():
+    browser_config = BrowserConfig()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com", config=run_config)
+        print(f"Success: {result.success} -> {result.url}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Environment Variable Configuration
+
+Load proxies from environment variables for easy configuration:
+
+```python
+import os
+from crawl4ai import ProxyConfig, CrawlerRunConfig
+
+# Set environment variable
+os.environ["PROXIES"] = "ip1:port1:user1:pass1,ip2:port2:user2:pass2,ip3:port3"
+
+# Load all proxies
+proxies = ProxyConfig.from_env()
+print(f"Loaded {len(proxies)} proxies")
+
+# Use first proxy
+if proxies:
+    run_config = CrawlerRunConfig(proxy_config=proxies[0])
+```
+
+## Rotating Proxies
+
+Crawl4AI supports automatic proxy rotation to distribute requests across multiple proxy servers. Rotation is applied per request using a rotation strategy on `CrawlerRunConfig`.
+
+### Proxy Rotation (recommended)
+```python
+import asyncio
+import re
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, ProxyConfig
+from crawl4ai.proxy_strategy import RoundRobinProxyStrategy
+
+async def main():
+    # Load proxies from environment
     proxies = ProxyConfig.from_env()
-    #eg: export PROXIES="ip1:port1:username1:password1,ip2:port2:username2:password2"
     if not proxies:
-        print("No proxies found in environment. Set PROXIES env variable!")
+        print("No proxies found! Set PROXIES environment variable.")
         return
 
+    # Create rotation strategy
     proxy_strategy = RoundRobinProxyStrategy(proxies)
 
-    # Create configs
+    # Configure per-request with proxy rotation
     browser_config = BrowserConfig(headless=True, verbose=False)
     run_config = CrawlerRunConfig(
         cache_mode=CacheMode.BYPASS,
-        proxy_rotation_strategy=proxy_strategy
+        proxy_rotation_strategy=proxy_strategy,
     )
 
     async with AsyncWebCrawler(config=browser_config) as crawler:
         urls = ["https://httpbin.org/ip"] * (len(proxies) * 2)  # Test each proxy twice
 
-        print("\n📈 Initializing crawler with proxy rotation...")
-        async with AsyncWebCrawler(config=browser_config) as crawler:
-            print("\n🚀 Starting batch crawl with proxy rotation...")
-            results = await crawler.arun_many(
-                urls=urls,
-                config=run_config
-            )
-            for result in results:
-                if result.success:
-                    ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
-                    current_proxy = run_config.proxy_config if run_config.proxy_config else None
+        print(f"🚀 Testing {len(proxies)} proxies with rotation...")
+        results = await crawler.arun_many(urls=urls, config=run_config)
 
-                    if current_proxy and ip_match:
-                        print(f"URL {result.url}")
-                        print(f"Proxy {current_proxy.server} -> Response IP: {ip_match.group(0)}")
-                        verified = ip_match.group(0) == current_proxy.ip
-                        if verified:
-                            print(f"✅ Proxy working! IP matches: {current_proxy.ip}")
-                        else:
-                            print("❌ Proxy failed or IP mismatch!")
-                    print("---")
+        for i, result in enumerate(results):
+            if result.success:
+                # Extract IP from response
+                ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
+                if ip_match:
+                    detected_ip = ip_match.group(0)
+                    proxy_index = i % len(proxies)
+                    expected_ip = proxies[proxy_index].ip
 
-asyncio.run(main())
+                    print(f"✅ Request {i+1}: Proxy {proxy_index+1} -> IP {detected_ip}")
+                    if detected_ip == expected_ip:
+                        print("   🎯 IP matches proxy configuration")
+                    else:
+                        print(f"   ⚠️  IP mismatch (expected {expected_ip})")
+                else:
+                    print(f"❌ Request {i+1}: Could not extract IP from response")
+            else:
+                print(f"❌ Request {i+1}: Failed - {result.error_message}")
 
+if __name__ == "__main__":
+    asyncio.run(main())
 ```
 
+## SSL Certificate Analysis
+
+Combine proxy usage with SSL certificate inspection for enhanced security analysis. SSL certificate fetching is configured per request via `CrawlerRunConfig`.
+
+### Per-Request SSL Certificate Analysis
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+run_config = CrawlerRunConfig(
+    proxy_config={
+        "server": "http://proxy.example.com:8080",
+        "username": "user",
+        "password": "pass",
+    },
+    fetch_ssl_certificate=True,  # Enable SSL certificate analysis for this request
+)
+
+
+async def main():
+    browser_config = BrowserConfig()
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        result = await crawler.arun(url="https://example.com", config=run_config)
+
+        if result.success:
+            print(f"✅ Crawled via proxy: {result.url}")
+
+            # Analyze SSL certificate
+            if result.ssl_certificate:
+                cert = result.ssl_certificate
+                print("🔒 SSL Certificate Info:")
+                print(f"   Issuer: {cert.issuer}")
+                print(f"   Subject: {cert.subject}")
+                print(f"   Valid until: {cert.valid_until}")
+                print(f"   Fingerprint: {cert.fingerprint}")
+
+                # Export certificate
+                cert.to_json("certificate.json")
+                print("💾 Certificate exported to certificate.json")
+            else:
+                print("⚠️  No SSL certificate information available")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Security Best Practices
+
+### 1. Proxy Rotation for Anonymity
+```python
+from crawl4ai import CrawlerRunConfig, ProxyConfig
+from crawl4ai.proxy_strategy import RoundRobinProxyStrategy
+
+# Use multiple proxies to avoid IP blocking
+proxies = ProxyConfig.from_env("PROXIES")
+strategy = RoundRobinProxyStrategy(proxies)
+
+# Configure rotation per request (recommended)
+run_config = CrawlerRunConfig(proxy_rotation_strategy=strategy)
+
+# For a fixed proxy across all requests, just reuse the same run_config instance
+static_run_config = run_config
+```
+
+### 2. SSL Certificate Verification
+```python
+from crawl4ai import CrawlerRunConfig
+
+# Always verify SSL certificates when possible
+# Per-request (affects specific requests)
+run_config = CrawlerRunConfig(fetch_ssl_certificate=True)
+```
+
+### 3. Environment Variable Security
+```bash
+# Use environment variables for sensitive proxy credentials
+# Avoid hardcoding usernames/passwords in code
+export PROXIES="ip1:port1:user1:pass1,ip2:port2:user2:pass2"
+```
+
+### 4. SOCKS5 for Enhanced Security
+```python
+from crawl4ai import CrawlerRunConfig
+
+# Prefer SOCKS5 proxies for better protocol support
+run_config = CrawlerRunConfig(proxy_config="socks5://proxy.example.com:1080")
+```
+
+## Migration from Deprecated `proxy` Parameter
+
+- "Deprecation Notice"
+    The legacy `proxy` argument on `BrowserConfig` is deprecated. Configure proxies through `CrawlerRunConfig.proxy_config` so each request fully describes its network settings.
+
+```python
+# Old (deprecated) approach
+# from crawl4ai import BrowserConfig
+# browser_config = BrowserConfig(proxy="http://proxy.example.com:8080")
+
+# New (preferred) approach
+from crawl4ai import CrawlerRunConfig
+run_config = CrawlerRunConfig(proxy_config="http://proxy.example.com:8080")
+```
+
+### Safe Logging of Proxies
+```python
+from crawl4ai import ProxyConfig
+
+def safe_proxy_repr(proxy: ProxyConfig):
+    if getattr(proxy, "username", None):
+        return f"{proxy.server} (auth: ****)"
+    return proxy.server
+```
+
+## Troubleshooting
+
+### Common Issues
+
+- "Proxy connection failed"
+    - Verify the proxy server is reachable from your network.
+    - Double-check authentication credentials.
+    - Ensure the protocol matches (`http`, `https`, or `socks5`).
+
+- "SSL certificate errors"
+    - Some proxies break SSL inspection; switch proxies if you see repeated failures.
+    - Consider temporarily disabling certificate fetching to isolate the issue.
+
+- "Environment variables not loading"
+    - Confirm `PROXIES` (or your custom env var) is set before running the script.
+    - Check formatting: `ip:port:user:pass,ip:port:user:pass`.
+
+- "Proxy rotation not working"
+    - Ensure `ProxyConfig.from_env()` actually loaded entries (`len(proxies) > 0`).
+    - Attach `proxy_rotation_strategy` to `CrawlerRunConfig`.
+    - Validate the proxy definitions you pass into the strategy.

docs/md_v2/api/parameters.md

@@ -21,21 +21,35 @@ browser_cfg = BrowserConfig(
 |-----------------------|----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
 | **`browser_type`**    | `"chromium"`, `"firefox"`, `"webkit"`<br/>*(default: `"chromium"`)* | Which browser engine to use. `"chromium"` is typical for many sites, `"firefox"` or `"webkit"` for specialized tests.                 |
 | **`headless`**        | `bool` (default: `True`)               | Headless means no visible UI. `False` is handy for debugging.                                                                         |
+| **`browser_mode`**    | `str` (default: `"dedicated"`)         | How browser is initialized: `"dedicated"` (new instance), `"builtin"` (CDP background), `"custom"` (explicit CDP), `"docker"` (container). |
+| **`use_managed_browser`** | `bool` (default: `False`)          | Launch browser via CDP for advanced control. Set automatically based on `browser_mode`.                  |
+| **`cdp_url`**         | `str` (default: `None`)                | Chrome DevTools Protocol endpoint URL (e.g., `"ws://localhost:9222/devtools/browser/"`). Set automatically based on `browser_mode`.   |
+| **`debugging_port`**  | `int` (default: `9222`)                | Port for browser debugging protocol.                                                                                                   |
+| **`host`**            | `str` (default: `"localhost"`)         | Host for browser connection.                                                                                                           |
 | **`viewport_width`**  | `int` (default: `1080`)                | Initial page width (in px). Useful for testing responsive layouts.                                                                    |
 | **`viewport_height`** | `int` (default: `600`)                 | Initial page height (in px).                                                                                                          |
+| **`viewport`**        | `dict` (default: `None`)               | Viewport dimensions dict. If set, overrides `viewport_width` and `viewport_height`.                                                   |
 | **`proxy`**           | `str` (deprecated)                      | Deprecated. Use `proxy_config` instead. If set, it will be auto-converted internally. |
-| **`proxy_config`**    | `dict` (default: `None`)               | For advanced or multi-proxy needs, specify details like `{"server": "...", "username": "...", ...}`.                                  |
+| **`proxy_config`**    | `ProxyConfig or dict` (default: `None`)| For advanced or multi-proxy needs, specify `ProxyConfig` object or dict like `{"server": "...", "username": "...", "password": "..."}`.  |
 | **`use_persistent_context`** | `bool` (default: `False`)       | If `True`, uses a **persistent** browser context (keep cookies, sessions across runs). Also sets `use_managed_browser=True`.          |
 | **`user_data_dir`**   | `str or None` (default: `None`)        | Directory to store user data (profiles, cookies). Must be set if you want permanent sessions.                                         |
+| **`chrome_channel`**  | `str` (default: `"chromium"`)          | Chrome channel to launch (e.g., "chrome", "msedge"). Only for `browser_type="chromium"`. Auto-set to empty for Firefox/WebKit.       |
+| **`channel`**         | `str` (default: `"chromium"`)          | Alias for `chrome_channel`.                                                                                                           |
+| **`accept_downloads`** | `bool` (default: `False`)             | Whether to allow file downloads. Requires `downloads_path` if `True`.                                                                 |
+| **`downloads_path`**  | `str or None` (default: `None`)        | Directory to store downloaded files.                                                                                                  |
+| **`storage_state`**   | `str or dict or None` (default: `None`)| In-memory storage state (cookies, localStorage) to restore browser state.                                                             |
 | **`ignore_https_errors`** | `bool` (default: `True`)           | If `True`, continues despite invalid certificates (common in dev/staging).                                                            |
 | **`java_script_enabled`** | `bool` (default: `True`)           | Disable if you want no JS overhead, or if only static content is needed.                                                              |
+| **`sleep_on_close`**  | `bool` (default: `False`)              | Add a small delay when closing browser (can help with cleanup issues).                                                                |
 | **`cookies`**         | `list` (default: `[]`)                 | Pre-set cookies, each a dict like `{"name": "session", "value": "...", "url": "..."}`.                                                |
 | **`headers`**         | `dict` (default: `{}`)                 | Extra HTTP headers for every request, e.g. `{"Accept-Language": "en-US"}`.                                                            |
-| **`user_agent`**      | `str` (default: Chrome-based UA)       | Your custom or random user agent. `user_agent_mode="random"` can shuffle it.                                                          |
-| **`light_mode`**      | `bool` (default: `False`)              | Disables some background features for performance gains.                                                                              |
+| **`user_agent`**      | `str` (default: Chrome-based UA)       | Your custom user agent string.                                                                                                        |
+| **`user_agent_mode`** | `str` (default: `""`)                  | Set to `"random"` to randomize user agent from a pool (helps with bot detection).                                                     |
+| **`user_agent_generator_config`** | `dict` (default: `{}`)     | Configuration dict for user agent generation when `user_agent_mode="random"`.                                                         |
 | **`text_mode`**       | `bool` (default: `False`)              | If `True`, tries to disable images/other heavy content for speed.                                                                     |
-| **`use_managed_browser`** | `bool` (default: `False`)          | For advanced “managed” interactions (debugging, CDP usage). Typically set automatically if persistent context is on.                  |
+| **`light_mode`**      | `bool` (default: `False`)              | Disables some background features for performance gains.                                                                              |
 | **`extra_args`**      | `list` (default: `[]`)                 | Additional flags for the underlying browser process, e.g. `["--disable-extensions"]`.                                                |
+| **`enable_stealth`**  | `bool` (default: `False`)              | Enable playwright-stealth mode to bypass bot detection. Cannot be used with `browser_mode="builtin"`.                                |
 
 **Tips**:
 - Set `headless=False` to visually **debug** how pages load or how interactions proceed.  
@@ -70,6 +84,7 @@ We group them by category.
 |------------------------------|--------------------------------------|-------------------------------------------------------------------------------------------------|
 | **`word_count_threshold`**   | `int` (default: ~200)                | Skips text blocks below X words. Helps ignore trivial sections.                                 |
 | **`extraction_strategy`**    | `ExtractionStrategy` (default: None) | If set, extracts structured data (CSS-based, LLM-based, etc.).                                  |
+| **`chunking_strategy`**      | `ChunkingStrategy` (default: RegexChunking()) | Strategy to chunk content before extraction. Can be customized for different chunking approaches. |
 | **`markdown_generator`**     | `MarkdownGenerationStrategy` (None)  | If you want specialized markdown output (citations, filtering, chunking, etc.). Can be customized with options such as `content_source` parameter to select the HTML input source ('cleaned_html', 'raw_html', or 'fit_html').                 |
 | **`css_selector`**           | `str` (None)                         | Retains only the part of the page matching this selector. Affects the entire extraction process. |
 | **`target_elements`**        | `List[str]` (None)                   | List of CSS selectors for elements to focus on for markdown generation and data extraction, while still processing the entire page for links, media, etc. Provides more flexibility than `css_selector`. |
@@ -78,32 +93,50 @@ We group them by category.
 | **`only_text`**              | `bool` (False)                       | If `True`, tries to extract text-only content.                                                  |
 | **`prettiify`**              | `bool` (False)                       | If `True`, beautifies final HTML (slower, purely cosmetic).                                      |
 | **`keep_data_attributes`**   | `bool` (False)                       | If `True`, preserve `data-*` attributes in cleaned HTML.                                         |
+| **`keep_attrs`**             | `list` (default: [])                 | List of HTML attributes to keep during processing (e.g., `["id", "class", "data-value"]`).      |
 | **`remove_forms`**           | `bool` (False)                       | If `True`, remove all `<form>` elements.                                                        |
+| **`parser_type`**            | `str` (default: "lxml")              | HTML parser to use (e.g., "lxml", "html.parser").                                               |
+| **`scraping_strategy`**      | `ContentScrapingStrategy` (default: LXMLWebScrapingStrategy()) | Strategy to use for content scraping. Can be customized for different scraping needs (e.g., PDF extraction). |
 
 ---
 
-### B) **Caching & Session**
+### B) **Browser Location and Identity**
+
+| **Parameter**          | **Type / Default**        | **What It Does**                                                                                       |
+|------------------------|---------------------------|--------------------------------------------------------------------------------------------------------|
+| **`locale`**           | `str or None` (None)      | Browser's locale (e.g., "en-US", "fr-FR") for language preferences.                                   |
+| **`timezone_id`**      | `str or None` (None)      | Browser's timezone (e.g., "America/New_York", "Europe/Paris").                                         |
+| **`geolocation`**      | `GeolocationConfig or None` (None) | GPS coordinates configuration. Use `GeolocationConfig(latitude=..., longitude=..., accuracy=...)`. |
+| **`fetch_ssl_certificate`** | `bool` (False)       | If `True`, fetches and includes SSL certificate information in the result.                             |
+| **`proxy_config`**           | `ProxyConfig or dict or None` (None) | Proxy configuration for this specific crawl. Can override browser-level proxy settings.          |
+| **`proxy_rotation_strategy`** | `ProxyRotationStrategy` (None)      | Strategy for rotating proxies during crawl operations.                                           |
+
+---
+
+### C) **Caching & Session**
 
 | **Parameter**           | **Type / Default**     | **What It Does**                                                                                                              |
 |-------------------------|------------------------|------------------------------------------------------------------------------------------------------------------------------|
 | **`cache_mode`**        | `CacheMode or None`    | Controls how caching is handled (`ENABLED`, `BYPASS`, `DISABLED`, etc.). If `None`, typically defaults to `ENABLED`.          |
 | **`session_id`**        | `str or None`          | Assign a unique ID to reuse a single browser session across multiple `arun()` calls.                                          |
-| **`bypass_cache`**      | `bool` (False)         | If `True`, acts like `CacheMode.BYPASS`.                                                                                     |
-| **`disable_cache`**     | `bool` (False)         | If `True`, acts like `CacheMode.DISABLED`.                                                                                   |
-| **`no_cache_read`**     | `bool` (False)         | If `True`, acts like `CacheMode.WRITE_ONLY` (writes cache but never reads).                                                  |
-| **`no_cache_write`**    | `bool` (False)         | If `True`, acts like `CacheMode.READ_ONLY` (reads cache but never writes).                                                   |
+| **`bypass_cache`**      | `bool` (False)         | **Deprecated.** If `True`, acts like `CacheMode.BYPASS`. Use `cache_mode` instead.                                           |
+| **`disable_cache`**     | `bool` (False)         | **Deprecated.** If `True`, acts like `CacheMode.DISABLED`. Use `cache_mode` instead.                                         |
+| **`no_cache_read`**     | `bool` (False)         | **Deprecated.** If `True`, acts like `CacheMode.WRITE_ONLY` (writes cache but never reads). Use `cache_mode` instead.        |
+| **`no_cache_write`**    | `bool` (False)         | **Deprecated.** If `True`, acts like `CacheMode.READ_ONLY` (reads cache but never writes). Use `cache_mode` instead.         |
+| **`shared_data`**       | `dict or None` (None)  | Shared data to be passed between hooks and accessible across crawl operations.                                                |
 
 Use these for controlling whether you read or write from a local content cache. Handy for large batch crawls or repeated site visits.
 
 ---
 
-### C) **Page Navigation & Timing**
+### D) **Page Navigation & Timing**
 
 | **Parameter**              | **Type / Default**      | **What It Does**                                                                                                    |
 |----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------|
-| **`wait_until`**           | `str` (domcontentloaded)| Condition for navigation to “complete”. Often `"networkidle"` or `"domcontentloaded"`.                               |
+| **`wait_until`**           | `str` (domcontentloaded)| Condition for navigation to "complete". Often `"networkidle"` or `"domcontentloaded"`.                               |
 | **`page_timeout`**         | `int` (60000 ms)        | Timeout for page navigation or JS steps. Increase for slow sites.                                                    |
 | **`wait_for`**             | `str or None`           | Wait for a CSS (`"css:selector"`) or JS (`"js:() => bool"`) condition before content extraction.                     |
+| **`wait_for_timeout`**     | `int or None` (None)    | Specific timeout in ms for the `wait_for` condition. If None, uses `page_timeout`.                                   |
 | **`wait_for_images`**      | `bool` (False)          | Wait for images to load before finishing. Slows down if you only want text.                                          |
 | **`delay_before_return_html`** | `float` (0.1)       | Additional pause (seconds) before final HTML is captured. Good for last-second updates.                               |
 | **`check_robots_txt`**     | `bool` (False)          | Whether to check and respect robots.txt rules before crawling. If True, caches robots.txt for efficiency.            |
@@ -112,15 +145,17 @@ Use these for controlling whether you read or write from a local content cache.
 
 ---
 
-### D) **Page Interaction**
+### E) **Page Interaction**
 
 | **Parameter**              | **Type / Default**            | **What It Does**                                                                                                                       |
 |----------------------------|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------|
 | **`js_code`**              | `str or list[str]` (None)      | JavaScript to run after load. E.g. `"document.querySelector('button')?.click();"`.                                                     |
-| **`js_only`**              | `bool` (False)                 | If `True`, indicates we’re reusing an existing session and only applying JS. No full reload.                                           |
+| **`c4a_script`**           | `str or list[str]` (None)      | C4A script that compiles to JavaScript. Alternative to writing raw JS.                                                                 |
+| **`js_only`**              | `bool` (False)                 | If `True`, indicates we're reusing an existing session and only applying JS. No full reload.                                           |
 | **`ignore_body_visibility`** | `bool` (True)                | Skip checking if `<body>` is visible. Usually best to keep `True`.                                                                     |
 | **`scan_full_page`**       | `bool` (False)                 | If `True`, auto-scroll the page to load dynamic content (infinite scroll).                                                              |
 | **`scroll_delay`**         | `float` (0.2)                  | Delay between scroll steps if `scan_full_page=True`.                                                                                   |
+| **`max_scroll_steps`**     | `int or None` (None)           | Maximum number of scroll steps during full page scan. If None, scrolls until entire page is loaded.                                     |
 | **`process_iframes`**      | `bool` (False)                 | Inlines iframe content for single-page extraction.                                                                                     |
 | **`remove_overlay_elements`** | `bool` (False)              | Removes potential modals/popups blocking the main content.                                                                              |
 | **`simulate_user`**        | `bool` (False)                 | Simulate user interactions (mouse movements) to avoid bot detection.                                                                    |
@@ -132,7 +167,7 @@ If your page is a single-page app with repeated JS updates, set `js_only=True` i
 
 ---
 
-### E) **Media Handling**
+### F) **Media Handling**
 
 | **Parameter**                              | **Type / Default**  | **What It Does**                                                                                         |
 |--------------------------------------------|---------------------|-----------------------------------------------------------------------------------------------------------|
@@ -141,13 +176,16 @@ If your page is a single-page app with repeated JS updates, set `js_only=True` i
 | **`screenshot_height_threshold`**          | `int` (~20000)      | If the page is taller than this, alternate screenshot strategies are used.                                |
 | **`pdf`**                                  | `bool` (False)      | If `True`, returns a PDF in `result.pdf`.                                                                 |
 | **`capture_mhtml`**                        | `bool` (False)      | If `True`, captures an MHTML snapshot of the page in `result.mhtml`. MHTML includes all page resources (CSS, images, etc.) in a single file. |
-| **`image_description_min_word_threshold`** | `int` (~50)         | Minimum words for an image’s alt text or description to be considered valid.                              |
+| **`image_description_min_word_threshold`** | `int` (~50)         | Minimum words for an image's alt text or description to be considered valid.                              |
 | **`image_score_threshold`**                | `int` (~3)          | Filter out low-scoring images. The crawler scores images by relevance (size, context, etc.).              |
 | **`exclude_external_images`**              | `bool` (False)      | Exclude images from other domains.                                                                        |
+| **`exclude_all_images`**                   | `bool` (False)      | If `True`, excludes all images from processing (both internal and external).                              |
+| **`table_score_threshold`**                | `int` (7)           | Minimum score threshold for processing a table. Lower values include more tables.                         |
+| **`table_extraction`**                     | `TableExtractionStrategy` (DefaultTableExtraction) | Strategy for table extraction. Defaults to DefaultTableExtraction with configured threshold. |
 
 ---
 
-### F) **Link/Domain Handling**
+### G) **Link/Domain Handling**
 
 | **Parameter**                | **Type / Default**      | **What It Does**                                                                                                             |
 |------------------------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------------|
@@ -155,23 +193,39 @@ If your page is a single-page app with repeated JS updates, set `js_only=True` i
 | **`exclude_external_links`** | `bool` (False)          | Removes all links pointing outside the current domain.                                                                      |
 | **`exclude_social_media_links`** | `bool` (False)      | Strips links specifically to social sites (like Facebook or Twitter).                                                      |
 | **`exclude_domains`**        | `list` ([])             | Provide a custom list of domains to exclude (like `["ads.com", "trackers.io"]`).                                            |
+| **`exclude_internal_links`** | `bool` (False)          | If `True`, excludes internal links from the results.                                                                        |
+| **`score_links`**            | `bool` (False)          | If `True`, calculates intrinsic quality scores for all links using URL structure, text quality, and contextual metrics.     |
 | **`preserve_https_for_internal_links`** | `bool` (False) | If `True`, preserves HTTPS scheme for internal links even when the server redirects to HTTP. Useful for security-conscious crawling. |
 
 Use these for link-level content filtering (often to keep crawls “internal” or to remove spammy domains).
 
 ---
 
-### G) **Debug & Logging**
+### H) **Debug, Logging & Network Monitoring**
 
 | **Parameter**  | **Type / Default** | **What It Does**                                                         |
 |----------------|--------------------|---------------------------------------------------------------------------|
 | **`verbose`**  | `bool` (True)     | Prints logs detailing each step of crawling, interactions, or errors.    |
-| **`log_console`** | `bool` (False) | Logs the page’s JavaScript console output if you want deeper JS debugging.|
+| **`log_console`** | `bool` (False) | Logs the page's JavaScript console output if you want deeper JS debugging.|
+| **`capture_network_requests`** | `bool` (False) | If `True`, captures network requests made by the page in `result.captured_requests`. |
+| **`capture_console_messages`** | `bool` (False) | If `True`, captures console messages from the page in `result.console_messages`. |
 
 ---
 
+### I) **Connection & HTTP Parameters**
 
-### H) **Virtual Scroll Configuration**
+| **Parameter**               | **Type / Default**      | **What It Does**                                                                                                    |
+|-----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------|
+| **`method`**                | `str` ("GET")          | HTTP method to use when using AsyncHTTPCrawlerStrategy (e.g., "GET", "POST").                                       |
+| **`stream`**                | `bool` (False)         | If `True`, enables streaming mode for `arun_many()` to process URLs as they complete rather than waiting for all.   |
+| **`url`**                   | `str or None` (None)   | URL for this specific config. Not typically set directly but used internally for URL-specific configurations.       |
+| **`user_agent`**            | `str or None` (None)   | Custom User-Agent string for this crawl. Can override browser-level user agent.                                     |
+| **`user_agent_mode`**       | `str or None` (None)   | Set to `"random"` to randomize user agent. Can override browser-level setting.                                      |
+| **`user_agent_generator_config`** | `dict` ({})      | Configuration for user agent generation when `user_agent_mode="random"`.                                            |
+
+---
+
+### J) **Virtual Scroll Configuration**
 
 | **Parameter**                | **Type / Default**           | **What It Does**                                                                                                                    |
 |------------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|
@@ -211,7 +265,7 @@ See [Virtual Scroll documentation](../../advanced/virtual-scroll.md) for detaile
 
 ---
 
-### I) **URL Matching Configuration**
+### K) **URL Matching Configuration**
 
 | **Parameter**          | **Type / Default**           | **What It Does**                                                                                                                    |
 |------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|
@@ -274,7 +328,25 @@ default_config = CrawlerRunConfig()  # No url_matcher = matches everything
 - If no config matches a URL and there's no default config (one without `url_matcher`), the URL will fail with "No matching configuration found"
 - Always include a default config as the last item if you want to handle all URLs
 
----## 2.2 Helper Methods
+---
+
+### L) **Advanced Crawling Features**
+
+| **Parameter**               | **Type / Default**           | **What It Does**                                                                                                                    |
+|-----------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|
+| **`deep_crawl_strategy`**   | `DeepCrawlStrategy or None` (None) | Strategy for deep/recursive crawling. Enables automatic link following and multi-level site crawling.                     |
+| **`link_preview_config`**   | `LinkPreviewConfig or dict or None` (None) | Configuration for link head extraction and scoring. Fetches and scores link metadata without full page loads.  |
+| **`experimental`**          | `dict or None` (None)       | Dictionary for experimental/beta features not yet integrated into main parameters. Use with caution.                                |
+
+**Deep Crawl Strategy** enables automatic site exploration by following links according to defined rules. Useful for sitemap generation or comprehensive site archiving.
+
+**Link Preview Config** allows efficient link discovery and scoring by fetching only the `<head>` section of linked pages, enabling smart crawl prioritization without the overhead of full page loads.
+
+**Experimental** parameters are features in beta testing. They may change or be removed in future versions. Check documentation for currently available experimental features.
+
+---
+
+## 2.2 Helper Methods
 
 Both `BrowserConfig` and `CrawlerRunConfig` provide a `clone()` method to create modified copies:
 
@@ -367,10 +439,19 @@ LLMConfig is useful to pass LLM provider config to strategies and functions that
 | **`provider`**    | `"ollama/llama3","groq/llama3-70b-8192","groq/llama3-8b-8192", "openai/gpt-4o-mini" ,"openai/gpt-4o","openai/o1-mini","openai/o1-preview","openai/o3-mini","openai/o3-mini-high","anthropic/claude-3-haiku-20240307","anthropic/claude-3-opus-20240229","anthropic/claude-3-sonnet-20240229","anthropic/claude-3-5-sonnet-20240620","gemini/gemini-pro","gemini/gemini-1.5-pro","gemini/gemini-2.0-flash","gemini/gemini-2.0-flash-exp","gemini/gemini-2.0-flash-lite-preview-02-05","deepseek/deepseek-chat"`<br/>*(default: `"openai/gpt-4o-mini"`)* | Which LLM provider to use. 
 | **`api_token`**         |1.Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`"GEMINI_API_KEY"` will be read from environment variables  <br/> 2. API token of LLM provider <br/> eg: `api_token = "gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv"` <br/> 3. Environment variable - use with prefix "env:" <br/> eg:`api_token = "env: GROQ_API_KEY"`              | API token to use for the given provider 
 | **`base_url`**         |Optional. Custom API endpoint | If your provider has a custom endpoint
+| **`backoff_base_delay`** |Optional. `int` *(default: `2`)* | Seconds to wait before the first retry when the provider throttles a request.
+| **`backoff_max_attempts`** |Optional. `int` *(default: `3`)* | Total tries (initial call + retries) before surfacing an error.
+| **`backoff_exponential_factor`** |Optional. `int` *(default: `2`)* | Multiplier that increases the wait time for each retry (`delay = base_delay * factor^attempt`).
 
 ## 3.2 Example Usage
 ```python
-llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
+llm_config = LLMConfig(
+    provider="openai/gpt-4o-mini",
+    api_token=os.getenv("OPENAI_API_KEY"),
+    backoff_base_delay=1, # optional
+    backoff_max_attempts=5, # optional
+    backoff_exponential_factor=3, # optional
+)
 ```
 
 ## 4. Putting It All Together

docs/md_v2/apps/c4a-script/README.md

@@ -18,7 +18,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip
 
 2. **Install Dependencies**
    ```bash
-   pip install flask
+   pip install -r requirements.txt
    ```
 
 3. **Launch the Server**
@@ -28,7 +28,7 @@ A comprehensive web-based tutorial for learning and experimenting with C4A-Scrip
 
 4. **Open in Browser**
    ```
-   http://localhost:8080
+   http://localhost:8000
    ```
 
 **🌐 Try Online**: [Live Demo](https://docs.crawl4ai.com/c4a-script/demo)
@@ -325,7 +325,7 @@ Powers the recording functionality:
 ### Configuration
 ```python
 # server.py configuration
-PORT = 8080
+PORT = 8000
 DEBUG = True
 THREADED = True
 ```
@@ -343,9 +343,9 @@ THREADED = True
 **Port Already in Use**
 ```bash
 # Kill existing process
-lsof -ti:8080 | xargs kill -9
+lsof -ti:8000 | xargs kill -9
 # Or use different port
-python server.py --port 8081
+python server.py --port 8001
 ```
 
 **Blockly Not Loading**

docs/md_v2/apps/c4a-script/server.py

@@ -216,7 +216,7 @@ def get_examples():
             'name': 'Handle Cookie Banner',
             'description': 'Accept cookies and close newsletter popup',
             'script': '''# Handle cookie banner and newsletter
-GO http://127.0.0.1:8080/playground/
+GO http://127.0.0.1:8000/playground/
 WAIT `body` 2
 IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
 IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`'''
@@ -283,7 +283,7 @@ WAIT `.success-message` 5'''
     return jsonify(examples)
 
 if __name__ == '__main__':
-    port = int(os.environ.get('PORT', 8080))
+    port = int(os.environ.get('PORT', 8000))
     print(f"""
 ╔══════════════════════════════════════════════════════════╗
 ║          C4A-Script Interactive Tutorial Server          ║

docs/md_v2/blog/index.md

@@ -20,22 +20,69 @@ Ever wondered why your AI coding assistant struggles with your library despite c
 
 ## Latest Release
 
-### [Crawl4AI v0.7.4 – The Intelligent Table Extraction & Performance Update](../blog/release-v0.7.4.md)
-*August 17, 2025*
+### [Crawl4AI v0.8.0 – Crash Recovery & Prefetch Mode](../blog/release-v0.8.0.md)
+*January 2026*
 
-Crawl4AI v0.7.4 introduces revolutionary LLM-powered table extraction with intelligent chunking, performance improvements for concurrent crawling, enhanced browser management, and critical stability fixes that make Crawl4AI more robust for production workloads.
+Crawl4AI v0.8.0 introduces crash recovery for deep crawls, a new prefetch mode for fast URL discovery, and critical security fixes for Docker deployments.
 
 Key highlights:
-- **🚀 LLMTableExtraction**: Revolutionary table extraction with intelligent chunking for massive tables
-- **⚡ Dispatcher Bug Fix**: Fixed sequential processing issue in arun_many for fast-completing tasks
-- **🧹 Memory Management Refactor**: Streamlined memory utilities and better resource management
-- **🔧 Browser Manager Fixes**: Resolved race conditions in concurrent page creation
-- **🔗 Advanced URL Processing**: Better handling of raw URLs and base tag link resolution
+- **🔄 Deep Crawl Crash Recovery**: `on_state_change` callback for real-time state persistence, `resume_state` to continue from checkpoints
+- **⚡ Prefetch Mode**: `prefetch=True` for 5-10x faster URL discovery, perfect for two-phase crawling patterns
+- **🔒 Security Fixes**: Hooks disabled by default, `file://` URLs blocked on Docker API, `__import__` removed from sandbox
 
-[Read full release notes →](../blog/release-v0.7.4.md)
+[Read full release notes →](../blog/release-v0.8.0.md)
+
+## Recent Releases
+
+### [Crawl4AI v0.7.8 – Stability & Bug Fix Release](../blog/release-v0.7.8.md)
+*December 2025*
+
+Crawl4AI v0.7.8 is a focused stability release addressing 11 bugs reported by the community. Fixes for Docker deployments, LLM extraction, URL handling, and dependency compatibility.
+
+Key highlights:
+- **🐳 Docker API Fixes**: ContentRelevanceFilter deserialization, ProxyConfig serialization, cache folder permissions
+- **🤖 LLM Improvements**: Configurable rate limiter backoff, HTML input format support
+- **📦 Dependencies**: Replaced deprecated PyPDF2 with pypdf, Pydantic v2 ConfigDict compatibility
+
+[Read full release notes →](../blog/release-v0.7.8.md)
+
+### [Crawl4AI v0.7.7 – The Self-Hosting & Monitoring Update](../blog/release-v0.7.7.md)
+*November 14, 2025*
+
+Crawl4AI v0.7.7 transforms Docker into a complete self-hosting platform with enterprise-grade real-time monitoring, comprehensive observability, and full operational control.
+
+Key highlights:
+- **📊 Real-time Monitoring Dashboard**: Interactive web UI with live system metrics
+- **🔌 Comprehensive Monitor API**: Complete REST API for programmatic access
+- **⚡ WebSocket Streaming**: Real-time updates every 2 seconds
+- **🔥 Smart Browser Pool**: 3-tier architecture with automatic promotion and cleanup
+
+[Read full release notes →](../blog/release-v0.7.7.md)
+
+### [Crawl4AI v0.7.6 – The Webhook Infrastructure Update](../blog/release-v0.7.6.md)
+*October 22, 2025*
+
+Crawl4AI v0.7.6 introduces comprehensive webhook support for the Docker job queue API, bringing real-time notifications to both crawling and LLM extraction workflows.
+
+Key highlights:
+- **🪝 Complete Webhook Support**: Real-time notifications for both `/crawl/job` and `/llm/job` endpoints
+- **🔄 Reliable Delivery**: Exponential backoff retry mechanism
+- **🔐 Custom Authentication**: Add custom headers for webhook authentication
+
+[Read full release notes →](../blog/release-v0.7.6.md)
 
 ---
 
+## Older Releases
+
+| Version | Date | Highlights |
+|---------|------|------------|
+| [v0.7.5](../blog/release-v0.7.5.md) | September 2025 | Docker Hooks System, enhanced LLM integration, HTTPS preservation |
+| [v0.7.4](../blog/release-v0.7.4.md) | August 2025 | LLM-powered table extraction, performance improvements |
+| [v0.7.3](../blog/release-v0.7.3.md) | July 2025 | Undetected browser, multi-URL config, memory monitoring |
+| [v0.7.1](../blog/release-v0.7.1.md) | June 2025 | Bug fixes and stability improvements |
+| [v0.7.0](../blog/release-v0.7.0.md) | May 2025 | Adaptive crawling, virtual scroll, link analysis |
+
 ## Project History
 
 Curious about how Crawl4AI has evolved? Check out our [complete changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md) for a detailed history of all versions and updates.

docs/md_v2/blog/releases/0.7.6.md

@@ -0,0 +1,314 @@
+# Crawl4AI v0.7.6 Release Notes
+
+*Release Date: October 22, 2025*
+
+I'm excited to announce Crawl4AI v0.7.6, featuring a complete webhook infrastructure for the Docker job queue API! This release eliminates polling and brings real-time notifications to both crawling and LLM extraction workflows.
+
+## 🎯 What's New
+
+### Webhook Support for Docker Job Queue API
+
+The headline feature of v0.7.6 is comprehensive webhook support for asynchronous job processing. No more constant polling to check if your jobs are done - get instant notifications when they complete!
+
+**Key Capabilities:**
+
+- ✅ **Universal Webhook Support**: Both `/crawl/job` and `/llm/job` endpoints now support webhooks
+- ✅ **Flexible Delivery Modes**: Choose notification-only or include full data in the webhook payload
+- ✅ **Reliable Delivery**: Exponential backoff retry mechanism (5 attempts: 1s → 2s → 4s → 8s → 16s)
+- ✅ **Custom Authentication**: Add custom headers for webhook authentication
+- ✅ **Global Configuration**: Set default webhook URL in `config.yml` for all jobs
+- ✅ **Task Type Identification**: Distinguish between `crawl` and `llm_extraction` tasks
+
+### How It Works
+
+Instead of constantly checking job status:
+
+**OLD WAY (Polling):**
+```python
+# Submit job
+response = requests.post("http://localhost:11235/crawl/job", json=payload)
+task_id = response.json()['task_id']
+
+# Poll until complete
+while True:
+    status = requests.get(f"http://localhost:11235/crawl/job/{task_id}")
+    if status.json()['status'] == 'completed':
+        break
+    time.sleep(5)  # Wait and try again
+```
+
+**NEW WAY (Webhooks):**
+```python
+# Submit job with webhook
+payload = {
+    "urls": ["https://example.com"],
+    "webhook_config": {
+        "webhook_url": "https://myapp.com/webhook",
+        "webhook_data_in_payload": True
+    }
+}
+response = requests.post("http://localhost:11235/crawl/job", json=payload)
+
+# Done! Webhook will notify you when complete
+# Your webhook handler receives the results automatically
+```
+
+### Crawl Job Webhooks
+
+```bash
+curl -X POST http://localhost:11235/crawl/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "urls": ["https://example.com"],
+    "browser_config": {"headless": true},
+    "crawler_config": {"cache_mode": "bypass"},
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/crawl-complete",
+      "webhook_data_in_payload": false,
+      "webhook_headers": {
+        "X-Webhook-Secret": "your-secret-token"
+      }
+    }
+  }'
+```
+
+### LLM Extraction Job Webhooks (NEW!)
+
+```bash
+curl -X POST http://localhost:11235/llm/job \
+  -H "Content-Type: application/json" \
+  -d '{
+    "url": "https://example.com/article",
+    "q": "Extract the article title, author, and publication date",
+    "schema": "{\"type\":\"object\",\"properties\":{\"title\":{\"type\":\"string\"}}}",
+    "provider": "openai/gpt-4o-mini",
+    "webhook_config": {
+      "webhook_url": "https://myapp.com/webhooks/llm-complete",
+      "webhook_data_in_payload": true
+    }
+  }'
+```
+
+### Webhook Payload Structure
+
+**Success (with data):**
+```json
+{
+  "task_id": "llm_1698765432",
+  "task_type": "llm_extraction",
+  "status": "completed",
+  "timestamp": "2025-10-22T10:30:00.000000+00:00",
+  "urls": ["https://example.com/article"],
+  "data": {
+    "extracted_content": {
+      "title": "Understanding Web Scraping",
+      "author": "John Doe",
+      "date": "2025-10-22"
+    }
+  }
+}
+```
+
+**Failure:**
+```json
+{
+  "task_id": "crawl_abc123",
+  "task_type": "crawl",
+  "status": "failed",
+  "timestamp": "2025-10-22T10:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "error": "Connection timeout after 30s"
+}
+```
+
+### Simple Webhook Handler Example
+
+```python
+from flask import Flask, request, jsonify
+
+app = Flask(__name__)
+
+@app.route('/webhook', methods=['POST'])
+def handle_webhook():
+    payload = request.json
+
+    task_id = payload['task_id']
+    task_type = payload['task_type']
+    status = payload['status']
+
+    if status == 'completed':
+        if 'data' in payload:
+            # Process data directly
+            data = payload['data']
+        else:
+            # Fetch from API
+            endpoint = 'crawl' if task_type == 'crawl' else 'llm'
+            response = requests.get(f'http://localhost:11235/{endpoint}/job/{task_id}')
+            data = response.json()
+
+        # Your business logic here
+        print(f"Job {task_id} completed!")
+
+    elif status == 'failed':
+        error = payload.get('error', 'Unknown error')
+        print(f"Job {task_id} failed: {error}")
+
+    return jsonify({"status": "received"}), 200
+
+app.run(port=8080)
+```
+
+## 📊 Performance Improvements
+
+- **Reduced Server Load**: Eliminates constant polling requests
+- **Lower Latency**: Instant notification vs. polling interval delay
+- **Better Resource Usage**: Frees up client connections while jobs run in background
+- **Scalable Architecture**: Handles high-volume crawling workflows efficiently
+
+## 🐛 Bug Fixes
+
+- Fixed webhook configuration serialization for Pydantic HttpUrl fields
+- Improved error handling in webhook delivery service
+- Enhanced Redis task storage for webhook config persistence
+
+## 🌍 Expected Real-World Impact
+
+### For Web Scraping Workflows
+- **Reduced Costs**: Less API calls = lower bandwidth and server costs
+- **Better UX**: Instant notifications improve user experience
+- **Scalability**: Handle 100s of concurrent jobs without polling overhead
+
+### For LLM Extraction Pipelines
+- **Async Processing**: Submit LLM extraction jobs and move on
+- **Batch Processing**: Queue multiple extractions, get notified as they complete
+- **Integration**: Easy integration with workflow automation tools (Zapier, n8n, etc.)
+
+### For Microservices
+- **Event-Driven**: Perfect for event-driven microservice architectures
+- **Decoupling**: Decouple job submission from result processing
+- **Reliability**: Automatic retries ensure webhooks are delivered
+
+## 🔄 Breaking Changes
+
+**None!** This release is fully backward compatible.
+
+- Webhook configuration is optional
+- Existing code continues to work without modification
+- Polling is still supported for jobs without webhook config
+
+## 📚 Documentation
+
+### New Documentation
+- **[WEBHOOK_EXAMPLES.md](../deploy/docker/WEBHOOK_EXAMPLES.md)** - Comprehensive webhook usage guide
+- **[docker_webhook_example.py](../docs/examples/docker_webhook_example.py)** - Working code examples
+
+### Updated Documentation
+- **[Docker README](../deploy/docker/README.md)** - Added webhook sections
+- API documentation with webhook examples
+
+## 🛠️ Migration Guide
+
+No migration needed! Webhooks are opt-in:
+
+1. **To use webhooks**: Add `webhook_config` to your job payload
+2. **To keep polling**: Continue using your existing code
+
+### Quick Start
+
+```python
+# Just add webhook_config to your existing payload
+payload = {
+    # Your existing configuration
+    "urls": ["https://example.com"],
+    "browser_config": {...},
+    "crawler_config": {...},
+
+    # NEW: Add webhook configuration
+    "webhook_config": {
+        "webhook_url": "https://myapp.com/webhook",
+        "webhook_data_in_payload": True
+    }
+}
+```
+
+## 🔧 Configuration
+
+### Global Webhook Configuration (config.yml)
+
+```yaml
+webhooks:
+  enabled: true
+  default_url: "https://myapp.com/webhooks/default"  # Optional
+  data_in_payload: false
+  retry:
+    max_attempts: 5
+    initial_delay_ms: 1000
+    max_delay_ms: 32000
+    timeout_ms: 30000
+  headers:
+    User-Agent: "Crawl4AI-Webhook/1.0"
+```
+
+## 🚀 Upgrade Instructions
+
+### Docker
+
+```bash
+# Pull the latest image
+docker pull unclecode/crawl4ai:0.7.6
+
+# Or use latest tag
+docker pull unclecode/crawl4ai:latest
+
+# Run with webhook support
+docker run -d \
+  -p 11235:11235 \
+  --env-file .llm.env \
+  --name crawl4ai \
+  unclecode/crawl4ai:0.7.6
+```
+
+### Python Package
+
+```bash
+pip install --upgrade crawl4ai
+```
+
+## 💡 Pro Tips
+
+1. **Use notification-only mode** for large results - fetch data separately to avoid large webhook payloads
+2. **Set custom headers** for webhook authentication and request tracking
+3. **Configure global default webhook** for consistent handling across all jobs
+4. **Implement idempotent webhook handlers** - same webhook may be delivered multiple times on retry
+5. **Use structured schemas** with LLM extraction for predictable webhook data
+
+## 🎬 Demo
+
+Try the release demo:
+
+```bash
+python docs/releases_review/demo_v0.7.6.py
+```
+
+This comprehensive demo showcases:
+- Crawl job webhooks (notification-only and with data)
+- LLM extraction webhooks (with JSON schema support)
+- Custom headers for authentication
+- Webhook retry mechanism
+- Real-time webhook receiver
+
+## 🙏 Acknowledgments
+
+Thank you to the community for the feedback that shaped this feature! Special thanks to everyone who requested webhook support for asynchronous job processing.
+
+## 📞 Support
+
+- **Documentation**: https://docs.crawl4ai.com
+- **GitHub Issues**: https://github.com/unclecode/crawl4ai/issues
+- **Discord**: https://discord.gg/crawl4ai
+
+---
+
+**Happy crawling with webhooks!** 🕷️🪝
+
+*- unclecode*

docs/md_v2/blog/releases/v0.7.5.md

@@ -0,0 +1,318 @@
+# 🚀 Crawl4AI v0.7.5: The Docker Hooks & Security Update
+
+*September 29, 2025 • 8 min read*
+
+---
+
+Today I'm releasing Crawl4AI v0.7.5—focused on extensibility and security. This update introduces the Docker Hooks System for pipeline customization, enhanced LLM integration, and important security improvements.
+
+## 🎯 What's New at a Glance
+
+- **Docker Hooks System**: Custom Python functions at key pipeline points with function-based API
+- **Function-Based Hooks**: New `hooks_to_string()` utility with Docker client auto-conversion
+- **Enhanced LLM Integration**: Custom providers with temperature control
+- **HTTPS Preservation**: Secure internal link handling
+- **Bug Fixes**: Resolved multiple community-reported issues
+- **Improved Docker Error Handling**: Better debugging and reliability
+
+## 🔧 Docker Hooks System: Pipeline Customization
+
+Every scraping project needs custom logic—authentication, performance optimization, content processing. Traditional solutions require forking or complex workarounds. Docker Hooks let you inject custom Python functions at 8 key points in the crawling pipeline.
+
+### Real Example: Authentication & Performance
+
+```python
+import requests
+
+# Real working hooks for httpbin.org
+hooks_config = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    print("Hook: Setting up page context")
+    # Block images to speed up crawling
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    print("Hook: Images blocked")
+    return page
+""",
+
+    "before_retrieve_html": """
+async def hook(page, context, **kwargs):
+    print("Hook: Before retrieving HTML")
+    # Scroll to bottom to load lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    print("Hook: Scrolled to bottom")
+    return page
+""",
+
+    "before_goto": """
+async def hook(page, context, url, **kwargs):
+    print(f"Hook: About to navigate to {url}")
+    # Add custom headers
+    await page.set_extra_http_headers({
+        'X-Test-Header': 'crawl4ai-hooks-test'
+    })
+    return page
+"""
+}
+
+# Test with Docker API
+payload = {
+    "urls": ["https://httpbin.org/html"],
+    "hooks": {
+        "code": hooks_config,
+        "timeout": 30
+    }
+}
+
+response = requests.post("http://localhost:11235/crawl", json=payload)
+result = response.json()
+
+if result.get('success'):
+    print("✅ Hooks executed successfully!")
+    print(f"Content length: {len(result.get('markdown', ''))} characters")
+```
+
+**Available Hook Points:**
+- `on_browser_created`: Browser setup
+- `on_page_context_created`: Page context configuration
+- `before_goto`: Pre-navigation setup
+- `after_goto`: Post-navigation processing
+- `on_user_agent_updated`: User agent changes
+- `on_execution_started`: Crawl initialization
+- `before_retrieve_html`: Pre-extraction processing
+- `before_return_html`: Final HTML processing
+
+### Function-Based Hooks API
+
+Writing hooks as strings works, but lacks IDE support and type checking. v0.7.5 introduces a function-based approach with automatic conversion!
+
+**Option 1: Using the `hooks_to_string()` Utility**
+
+```python
+from crawl4ai import hooks_to_string
+import requests
+
+# Define hooks as regular Python functions (with full IDE support!)
+async def on_page_context_created(page, context, **kwargs):
+    """Block images to speed up crawling"""
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+
+async def before_goto(page, context, url, **kwargs):
+    """Add custom headers"""
+    await page.set_extra_http_headers({
+        'X-Crawl4AI': 'v0.7.5',
+        'X-Custom-Header': 'my-value'
+    })
+    return page
+
+# Convert functions to strings
+hooks_code = hooks_to_string({
+    "on_page_context_created": on_page_context_created,
+    "before_goto": before_goto
+})
+
+# Use with REST API
+payload = {
+    "urls": ["https://httpbin.org/html"],
+    "hooks": {"code": hooks_code, "timeout": 30}
+}
+response = requests.post("http://localhost:11235/crawl", json=payload)
+```
+
+**Option 2: Docker Client with Automatic Conversion (Recommended!)**
+
+```python
+from crawl4ai.docker_client import Crawl4aiDockerClient
+
+# Define hooks as functions (same as above)
+async def on_page_context_created(page, context, **kwargs):
+    await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
+    return page
+
+async def before_retrieve_html(page, context, **kwargs):
+    # Scroll to load lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+    return page
+
+# Use Docker client - conversion happens automatically!
+client = Crawl4aiDockerClient(base_url="http://localhost:11235")
+
+results = await client.crawl(
+    urls=["https://httpbin.org/html"],
+    hooks={
+        "on_page_context_created": on_page_context_created,
+        "before_retrieve_html": before_retrieve_html
+    },
+    hooks_timeout=30
+)
+
+if results and results.success:
+    print(f"✅ Hooks executed! HTML length: {len(results.html)}")
+```
+
+**Benefits of Function-Based Hooks:**
+- ✅ Full IDE support (autocomplete, syntax highlighting)
+- ✅ Type checking and linting
+- ✅ Easier to test and debug
+- ✅ Reusable across projects
+- ✅ Automatic conversion in Docker client
+- ✅ No breaking changes - string hooks still work!
+
+## 🤖 Enhanced LLM Integration
+
+Enhanced LLM integration with custom providers, temperature control, and base URL configuration.
+
+### Multi-Provider Support
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+# Test with different providers
+async def test_llm_providers():
+    # OpenAI with custom temperature
+    openai_strategy = LLMExtractionStrategy(
+        provider="gemini/gemini-2.5-flash-lite",
+        api_token="your-api-token",
+        temperature=0.7,  # New in v0.7.5
+        instruction="Summarize this page in one sentence"
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            "https://example.com",
+            config=CrawlerRunConfig(extraction_strategy=openai_strategy)
+        )
+
+        if result.success:
+            print("✅ LLM extraction completed")
+            print(result.extracted_content)
+
+# Docker API with enhanced LLM config
+llm_payload = {
+    "url": "https://example.com",
+    "f": "llm",
+    "q": "Summarize this page in one sentence.",
+    "provider": "gemini/gemini-2.5-flash-lite",
+    "temperature": 0.7
+}
+
+response = requests.post("http://localhost:11235/md", json=llm_payload)
+```
+
+**New Features:**
+- Custom `temperature` parameter for creativity control
+- `base_url` for custom API endpoints
+- Multi-provider environment variable support
+- Docker API integration
+
+## 🔒 HTTPS Preservation
+
+**The Problem:** Modern web apps require HTTPS everywhere. When crawlers downgrade internal links from HTTPS to HTTP, authentication breaks and security warnings appear.
+
+**Solution:** HTTPS preservation maintains secure protocols throughout crawling.
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, FilterChain, URLPatternFilter, BFSDeepCrawlStrategy
+
+async def test_https_preservation():
+    # Enable HTTPS preservation
+    url_filter = URLPatternFilter(
+        patterns=["^(https:\/\/)?quotes\.toscrape\.com(\/.*)?$"]
+    )
+
+    config = CrawlerRunConfig(
+        exclude_external_links=True,
+        preserve_https_for_internal_links=True,  # New in v0.7.5
+        deep_crawl_strategy=BFSDeepCrawlStrategy(
+            max_depth=2,
+            max_pages=5,
+            filter_chain=FilterChain([url_filter])
+        )
+    )
+
+    async with AsyncWebCrawler() as crawler:
+        async for result in await crawler.arun(
+            url="https://quotes.toscrape.com",
+            config=config
+        ):
+            # All internal links maintain HTTPS
+            internal_links = [link['href'] for link in result.links['internal']]
+            https_links = [link for link in internal_links if link.startswith('https://')]
+
+            print(f"HTTPS links preserved: {len(https_links)}/{len(internal_links)}")
+            for link in https_links[:3]:
+                print(f"  → {link}")
+```
+
+## 🛠️ Bug Fixes and Improvements
+
+### Major Fixes
+- **URL Processing**: Fixed '+' sign preservation in query parameters (#1332)
+- **Proxy Configuration**: Enhanced proxy string parsing (old `proxy` parameter deprecated)
+- **Docker Error Handling**: Comprehensive error messages with status codes
+- **Memory Management**: Fixed leaks in long-running sessions
+- **JWT Authentication**: Fixed Docker JWT validation issues (#1442)
+- **Playwright Stealth**: Fixed stealth features for Playwright integration (#1481)
+- **API Configuration**: Fixed config handling to prevent overriding user-provided settings (#1505)
+- **Docker Filter Serialization**: Resolved JSON encoding errors in deep crawl strategy (#1419)
+- **LLM Provider Support**: Fixed custom LLM provider integration for adaptive crawler (#1291)
+- **Performance Issues**: Resolved backoff strategy failures and timeout handling (#989)
+
+### Community-Reported Issues Fixed
+This release addresses multiple issues reported by the community through GitHub issues and Discord discussions:
+- Fixed browser configuration reference errors
+- Resolved dependency conflicts with cssselect
+- Improved error messaging for failed authentications
+- Enhanced compatibility with various proxy configurations
+- Fixed edge cases in URL normalization
+
+### Configuration Updates
+```python
+# Old proxy config (deprecated)
+# browser_config = BrowserConfig(proxy="http://proxy:8080")
+
+# New enhanced proxy config
+browser_config = BrowserConfig(
+    proxy_config={
+        "server": "http://proxy:8080",
+        "username": "optional-user",
+        "password": "optional-pass"
+    }
+)
+```
+
+## 🔄 Breaking Changes
+
+1. **Python 3.10+ Required**: Upgrade from Python 3.9
+2. **Proxy Parameter Deprecated**: Use new `proxy_config` structure
+3. **New Dependency**: Added `cssselect` for better CSS handling
+
+## 🚀 Get Started
+
+```bash
+# Install latest version
+pip install crawl4ai==0.7.5
+
+# Docker deployment
+docker pull unclecode/crawl4ai:latest
+docker run -p 11235:11235 unclecode/crawl4ai:latest
+```
+
+**Try the Demo:**
+```bash
+# Run working examples
+python docs/releases_review/demo_v0.7.5.py
+```
+
+**Resources:**
+- 📖 Documentation: [docs.crawl4ai.com](https://docs.crawl4ai.com)
+- 🐙 GitHub: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
+- 💬 Discord: [discord.gg/crawl4ai](https://discord.gg/jP8KfhDhyN)
+- 🐦 Twitter: [@unclecode](https://x.com/unclecode)
+
+Happy crawling! 🕷️

docs/md_v2/blog/releases/v0.7.7.md

@@ -0,0 +1,626 @@
+# 🚀 Crawl4AI v0.7.7: The Self-Hosting & Monitoring Update
+
+*November 14, 2025 • 10 min read*
+
+---
+
+Today I'm releasing Crawl4AI v0.7.7—the Self-Hosting & Monitoring Update. This release transforms Crawl4AI Docker from a simple containerized crawler into a complete self-hosting platform with enterprise-grade real-time monitoring, full operational transparency, and production-ready observability.
+
+## 🎯 What's New at a Glance
+
+- **📊 Real-time Monitoring Dashboard**: Interactive web UI with live system metrics and browser pool status
+- **🔌 Comprehensive Monitor API**: Complete REST API for programmatic access to all monitoring data
+- **⚡ WebSocket Streaming**: Real-time updates every 2 seconds for custom dashboards
+- **🎮 Control Actions**: Manual browser management (kill, restart, cleanup)
+- **🔥 Smart Browser Pool**: 3-tier architecture (permanent/hot/cold) with automatic promotion
+- **🧹 Janitor Cleanup System**: Automatic resource management with event logging
+- **📈 Production Metrics**: 6 critical metrics for operational excellence
+- **🏭 Integration Ready**: Prometheus, alerting, and log aggregation examples
+- **🐛 Critical Bug Fixes**: Async LLM extraction, DFS crawling, viewport config, and more
+
+## 📊 Real-time Monitoring Dashboard: Complete Visibility
+
+**The Problem:** Running Crawl4AI in Docker was like flying blind. Users had no visibility into what was happening inside the container—memory usage, active requests, browser pools, or errors. Troubleshooting required checking logs, and there was no way to monitor performance or manually intervene when issues occurred.
+
+**My Solution:** I built a complete real-time monitoring system with an interactive dashboard, comprehensive REST API, WebSocket streaming, and manual control actions. Now you have full transparency and control over your crawling infrastructure.
+
+### The Self-Hosting Value Proposition
+
+Before v0.7.7, Docker was just a containerized crawler. After v0.7.7, it's a complete self-hosting platform that gives you:
+
+- **🔒 Data Privacy**: Your data never leaves your infrastructure
+- **💰 Cost Control**: No per-request pricing or rate limits
+- **🎯 Full Customization**: Complete control over configurations and strategies
+- **📊 Complete Transparency**: Real-time visibility into every aspect
+- **⚡ Performance**: Direct access without network overhead
+- **🛡️ Enterprise Security**: Keep workflows behind your firewall
+
+### Interactive Monitoring Dashboard
+
+Access the dashboard at `http://localhost:11235/dashboard` to see:
+
+- **System Health Overview**: CPU, memory, network, and uptime in real-time
+- **Live Request Tracking**: Active and completed requests with full details
+- **Browser Pool Management**: Interactive table with permanent/hot/cold browsers
+- **Janitor Events Log**: Automatic cleanup activities
+- **Error Monitoring**: Full context error logs
+
+The dashboard updates every 2 seconds via WebSocket, giving you live visibility into your crawling operations.
+
+## 🔌 Monitor API: Programmatic Access
+
+**The Problem:** Monitoring dashboards are great for humans, but automation and integration require programmatic access.
+
+**My Solution:** A comprehensive REST API that exposes all monitoring data for integration with your existing infrastructure.
+
+### System Health Endpoint
+
+```python
+import httpx
+import asyncio
+
+async def monitor_system_health():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/health")
+        health = response.json()
+
+        print(f"Container Metrics:")
+        print(f"  CPU: {health['container']['cpu_percent']:.1f}%")
+        print(f"  Memory: {health['container']['memory_percent']:.1f}%")
+        print(f"  Uptime: {health['container']['uptime_seconds']}s")
+
+        print(f"\nBrowser Pool:")
+        print(f"  Permanent: {health['pool']['permanent']['active']} active")
+        print(f"  Hot Pool: {health['pool']['hot']['count']} browsers")
+        print(f"  Cold Pool: {health['pool']['cold']['count']} browsers")
+
+        print(f"\nStatistics:")
+        print(f"  Total Requests: {health['stats']['total_requests']}")
+        print(f"  Success Rate: {health['stats']['success_rate_percent']:.1f}%")
+        print(f"  Avg Latency: {health['stats']['avg_latency_ms']:.0f}ms")
+
+asyncio.run(monitor_system_health())
+```
+
+### Request Tracking
+
+```python
+async def track_requests():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/requests")
+        requests_data = response.json()
+
+        print(f"Active Requests: {len(requests_data['active'])}")
+        print(f"Completed Requests: {len(requests_data['completed'])}")
+
+        # See details of recent requests
+        for req in requests_data['completed'][:5]:
+            status_icon = "✅" if req['success'] else "❌"
+            print(f"{status_icon} {req['endpoint']} - {req['latency_ms']:.0f}ms")
+```
+
+### Browser Pool Management
+
+```python
+async def monitor_browser_pool():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/browsers")
+        browsers = response.json()
+
+        print(f"Pool Summary:")
+        print(f"  Total Browsers: {browsers['summary']['total_count']}")
+        print(f"  Total Memory: {browsers['summary']['total_memory_mb']} MB")
+        print(f"  Reuse Rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
+
+        # List all browsers
+        for browser in browsers['permanent']:
+            print(f"🔥 Permanent: {browser['browser_id'][:8]}... | "
+                  f"Requests: {browser['request_count']} | "
+                  f"Memory: {browser['memory_mb']:.0f} MB")
+```
+
+### Endpoint Performance Statistics
+
+```python
+async def get_endpoint_stats():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/endpoints/stats")
+        stats = response.json()
+
+        print("Endpoint Analytics:")
+        for endpoint, data in stats.items():
+            print(f"  {endpoint}:")
+            print(f"    Requests: {data['count']}")
+            print(f"    Avg Latency: {data['avg_latency_ms']:.0f}ms")
+            print(f"    Success Rate: {data['success_rate_percent']:.1f}%")
+```
+
+### Complete API Reference
+
+The Monitor API includes these endpoints:
+
+- `GET /monitor/health` - System health with pool statistics
+- `GET /monitor/requests` - Active and completed request tracking
+- `GET /monitor/browsers` - Browser pool details and efficiency
+- `GET /monitor/endpoints/stats` - Per-endpoint performance analytics
+- `GET /monitor/timeline?minutes=5` - Time-series data for charts
+- `GET /monitor/logs/janitor?limit=10` - Cleanup activity logs
+- `GET /monitor/logs/errors?limit=10` - Error logs with context
+- `POST /monitor/actions/cleanup` - Force immediate cleanup
+- `POST /monitor/actions/kill_browser` - Kill specific browser
+- `POST /monitor/actions/restart_browser` - Restart browser
+- `POST /monitor/stats/reset` - Reset accumulated statistics
+
+## ⚡ WebSocket Streaming: Real-time Updates
+
+**The Problem:** Polling the API every few seconds wastes resources and adds latency. Real-time dashboards need instant updates.
+
+**My Solution:** WebSocket streaming with 2-second update intervals for building custom real-time dashboards.
+
+### WebSocket Integration Example
+
+```python
+import websockets
+import json
+import asyncio
+
+async def monitor_realtime():
+    uri = "ws://localhost:11235/monitor/ws"
+
+    async with websockets.connect(uri) as websocket:
+        print("Connected to real-time monitoring stream")
+
+        while True:
+            # Receive update every 2 seconds
+            data = await websocket.recv()
+            update = json.loads(data)
+
+            # Access all monitoring data
+            print(f"\n--- Update at {update['timestamp']} ---")
+            print(f"Memory: {update['health']['container']['memory_percent']:.1f}%")
+            print(f"Active Requests: {len(update['requests']['active'])}")
+            print(f"Total Browsers: {update['browsers']['summary']['total_count']}")
+
+            if update['errors']:
+                print(f"⚠️  Recent Errors: {len(update['errors'])}")
+
+asyncio.run(monitor_realtime())
+```
+
+**Expected Real-World Impact:**
+- **Custom Dashboards**: Build tailored monitoring UIs for your team
+- **Real-time Alerting**: Trigger alerts instantly when metrics exceed thresholds
+- **Integration**: Feed live data into monitoring tools like Grafana
+- **Automation**: React to events in real-time without polling
+
+## 🔥 Smart Browser Pool: 3-Tier Architecture
+
+**The Problem:** Creating a new browser for every request is slow and memory-intensive. Traditional browser pools are static and inefficient.
+
+**My Solution:** A smart 3-tier browser pool that automatically adapts to usage patterns.
+
+### How It Works
+
+```python
+import httpx
+
+async def demonstrate_browser_pool():
+    async with httpx.AsyncClient() as client:
+        # Request 1-3: Default config → Uses permanent browser
+        print("Phase 1: Using permanent browser")
+        for i in range(3):
+            await client.post(
+                "http://localhost:11235/crawl",
+                json={"urls": [f"https://httpbin.org/html?req={i}"]}
+            )
+            print(f"  Request {i+1}: Reused permanent browser")
+
+        # Request 4-6: Custom viewport → Cold pool (first use)
+        print("\nPhase 2: Custom config creates cold pool browser")
+        viewport_config = {"viewport": {"width": 1280, "height": 720}}
+        for i in range(4):
+            await client.post(
+                "http://localhost:11235/crawl",
+                json={
+                    "urls": [f"https://httpbin.org/json?v={i}"],
+                    "browser_config": viewport_config
+                }
+            )
+            if i < 2:
+                print(f"  Request {i+1}: Cold pool browser")
+            else:
+                print(f"  Request {i+1}: Promoted to hot pool! (after 3 uses)")
+
+        # Check pool status
+        response = await client.get("http://localhost:11235/monitor/browsers")
+        browsers = response.json()
+
+        print(f"\nPool Status:")
+        print(f"  Permanent: {len(browsers['permanent'])} (always active)")
+        print(f"  Hot: {len(browsers['hot'])} (frequently used configs)")
+        print(f"  Cold: {len(browsers['cold'])} (on-demand)")
+        print(f"  Reuse Rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
+
+asyncio.run(demonstrate_browser_pool())
+```
+
+**Pool Tiers:**
+
+- **🔥 Permanent Browser**: Always-on, default configuration, instant response
+- **♨️ Hot Pool**: Browsers promoted after 3+ uses, kept warm for quick access
+- **❄️ Cold Pool**: On-demand browsers for variant configs, cleaned up when idle
+
+**Expected Real-World Impact:**
+- **Memory Efficiency**: 10x reduction in memory usage vs creating browsers per request
+- **Performance**: Instant access to frequently-used configurations
+- **Automatic Optimization**: Pool adapts to your usage patterns
+- **Resource Management**: Janitor automatically cleans up idle browsers
+
+## 🧹 Janitor System: Automatic Cleanup
+
+**The Problem:** Long-running crawlers accumulate idle browsers and consume memory over time.
+
+**My Solution:** An automatic janitor system that monitors and cleans up idle resources.
+
+```python
+async def monitor_janitor_activity():
+    async with httpx.AsyncClient() as client:
+        response = await client.get("http://localhost:11235/monitor/logs/janitor?limit=5")
+        logs = response.json()
+
+        print("Recent Cleanup Activities:")
+        for log in logs:
+            print(f"  {log['timestamp']}: {log['message']}")
+
+# Example output:
+# 2025-11-14 10:30:00: Cleaned up 2 cold pool browsers (idle > 5min)
+# 2025-11-14 10:25:00: Browser reuse rate: 85.3%
+# 2025-11-14 10:20:00: Hot pool browser promoted (10 requests)
+```
+
+## 🎮 Control Actions: Manual Management
+
+**The Problem:** Sometimes you need to manually intervene—kill a stuck browser, force cleanup, or restart resources.
+
+**My Solution:** Manual control actions via the API for operational troubleshooting.
+
+### Force Cleanup
+
+```python
+async def force_cleanup():
+    async with httpx.AsyncClient() as client:
+        response = await client.post("http://localhost:11235/monitor/actions/cleanup")
+        result = response.json()
+
+        print(f"Cleanup completed:")
+        print(f"  Browsers cleaned: {result.get('cleaned_count', 0)}")
+        print(f"  Memory freed: {result.get('memory_freed_mb', 0):.1f} MB")
+```
+
+### Kill Specific Browser
+
+```python
+async def kill_stuck_browser(browser_id: str):
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            "http://localhost:11235/monitor/actions/kill_browser",
+            json={"browser_id": browser_id}
+        )
+
+        if response.status_code == 200:
+            print(f"✅ Browser {browser_id} killed successfully")
+```
+
+### Reset Statistics
+
+```python
+async def reset_stats():
+    async with httpx.AsyncClient() as client:
+        response = await client.post("http://localhost:11235/monitor/stats/reset")
+        print("📊 Statistics reset for fresh monitoring")
+```
+
+## 📈 Production Integration Patterns
+
+### Prometheus Integration
+
+```python
+# Export metrics for Prometheus scraping
+async def export_prometheus_metrics():
+    async with httpx.AsyncClient() as client:
+        health = await client.get("http://localhost:11235/monitor/health")
+        data = health.json()
+
+        # Export in Prometheus format
+        metrics = f"""
+# HELP crawl4ai_memory_usage_percent Memory usage percentage
+# TYPE crawl4ai_memory_usage_percent gauge
+crawl4ai_memory_usage_percent {data['container']['memory_percent']}
+
+# HELP crawl4ai_request_success_rate Request success rate
+# TYPE crawl4ai_request_success_rate gauge
+crawl4ai_request_success_rate {data['stats']['success_rate_percent']}
+
+# HELP crawl4ai_browser_pool_count Total browsers in pool
+# TYPE crawl4ai_browser_pool_count gauge
+crawl4ai_browser_pool_count {data['pool']['permanent']['active'] + data['pool']['hot']['count'] + data['pool']['cold']['count']}
+"""
+        return metrics
+```
+
+### Alerting Example
+
+```python
+async def check_alerts():
+    async with httpx.AsyncClient() as client:
+        health = await client.get("http://localhost:11235/monitor/health")
+        data = health.json()
+
+        # Memory alert
+        if data['container']['memory_percent'] > 80:
+            print("🚨 ALERT: Memory usage above 80%")
+            # Trigger cleanup
+            await client.post("http://localhost:11235/monitor/actions/cleanup")
+
+        # Success rate alert
+        if data['stats']['success_rate_percent'] < 90:
+            print("🚨 ALERT: Success rate below 90%")
+            # Check error logs
+            errors = await client.get("http://localhost:11235/monitor/logs/errors")
+            print(f"Recent errors: {len(errors.json())}")
+
+        # Latency alert
+        if data['stats']['avg_latency_ms'] > 5000:
+            print("🚨 ALERT: Average latency above 5s")
+```
+
+### Key Metrics to Track
+
+```python
+CRITICAL_METRICS = {
+    "memory_usage": {
+        "current": "container.memory_percent",
+        "target": "<80%",
+        "alert_threshold": ">80%",
+        "action": "Force cleanup or scale"
+    },
+    "success_rate": {
+        "current": "stats.success_rate_percent",
+        "target": ">95%",
+        "alert_threshold": "<90%",
+        "action": "Check error logs"
+    },
+    "avg_latency": {
+        "current": "stats.avg_latency_ms",
+        "target": "<2000ms",
+        "alert_threshold": ">5000ms",
+        "action": "Investigate slow requests"
+    },
+    "browser_reuse_rate": {
+        "current": "browsers.summary.reuse_rate_percent",
+        "target": ">80%",
+        "alert_threshold": "<60%",
+        "action": "Check pool configuration"
+    },
+    "total_browsers": {
+        "current": "browsers.summary.total_count",
+        "target": "<15",
+        "alert_threshold": ">20",
+        "action": "Check for browser leaks"
+    },
+    "error_frequency": {
+        "current": "len(errors)",
+        "target": "<5/hour",
+        "alert_threshold": ">10/hour",
+        "action": "Review error patterns"
+    }
+}
+```
+
+## 🐛 Critical Bug Fixes
+
+This release includes significant bug fixes that improve stability and performance:
+
+### Async LLM Extraction (#1590)
+
+**The Problem:** LLM extraction was blocking async execution, causing URLs to be processed sequentially instead of in parallel (issue #1055).
+
+**The Fix:** Resolved the blocking issue to enable true parallel processing for LLM extraction.
+
+```python
+# Before v0.7.7: Sequential processing
+# After v0.7.7: True parallel processing
+
+async with AsyncWebCrawler() as crawler:
+    urls = ["url1", "url2", "url3", "url4"]
+
+    # Now processes truly in parallel with LLM extraction
+    results = await crawler.arun_many(
+        urls,
+        config=CrawlerRunConfig(
+            extraction_strategy=LLMExtractionStrategy(...)
+        )
+    )
+    # 4x faster for parallel LLM extraction!
+```
+
+**Expected Impact:** Major performance improvement for batch LLM extraction workflows.
+
+### DFS Deep Crawling (#1607)
+
+**The Problem:** DFS (Depth-First Search) deep crawl strategy had implementation issues.
+
+**The Fix:** Enhanced DFSDeepCrawlStrategy with proper seen URL tracking and improved documentation.
+
+### Browser & Crawler Config Documentation (#1609)
+
+**The Problem:** Documentation didn't match the actual `async_configs.py` implementation.
+
+**The Fix:** Updated all configuration documentation to accurately reflect the current implementation.
+
+### Sitemap Seeder (#1598)
+
+**The Problem:** Sitemap parsing and URL normalization issues in AsyncUrlSeeder (issue #1559).
+
+**The Fix:** Added comprehensive tests and fixes for sitemap namespace parsing and URL normalization.
+
+### Remove Overlay Elements (#1529)
+
+**The Problem:** The `remove_overlay_elements` functionality wasn't working (issue #1396).
+
+**The Fix:** Fixed by properly calling the injected JavaScript function.
+
+### Viewport Configuration (#1495)
+
+**The Problem:** Viewport configuration wasn't working in managed browsers (issue #1490).
+
+**The Fix:** Added proper viewport size configuration support for browser launch.
+
+### Managed Browser CDP Timing (#1528)
+
+**The Problem:** CDP (Chrome DevTools Protocol) endpoint verification had timing issues causing connection failures (issue #1445).
+
+**The Fix:** Added exponential backoff for CDP endpoint verification to handle timing variations.
+
+### Security Updates
+
+- **pyOpenSSL**: Updated from >=24.3.0 to >=25.3.0 to address security vulnerability
+- Added verification tests for the security update
+
+### Docker Fixes
+
+- **Port Standardization**: Fixed inconsistent port usage (11234 vs 11235) - now standardized to 11235
+- **LLM Environment**: Fixed LLM API key handling for multi-provider support (PR #1537)
+- **Error Handling**: Improved Docker API error messages with comprehensive status codes
+- **Serialization**: Fixed `fit_html` property serialization in `/crawl` and `/crawl/stream` endpoints
+
+### Other Important Fixes
+
+- **arun_many Returns**: Fixed function to always return a list, even on exception (PR #1530)
+- **Webhook Serialization**: Properly serialize Pydantic HttpUrl in webhook config
+- **LLMConfig Documentation**: Fixed casing and variable name consistency (issue #1551)
+- **Python Version**: Dropped Python 3.9 support, now requires Python >=3.10
+
+## 📊 Expected Real-World Impact
+
+### For DevOps & Infrastructure Teams
+- **Full Visibility**: Know exactly what's happening inside your crawling infrastructure
+- **Proactive Monitoring**: Catch issues before they become problems
+- **Resource Optimization**: Identify memory leaks and performance bottlenecks
+- **Operational Control**: Manual intervention when automated systems need help
+
+### For Production Deployments
+- **Enterprise Observability**: Prometheus, Grafana, and alerting integration
+- **Debugging**: Real-time logs and error tracking
+- **Capacity Planning**: Historical metrics for scaling decisions
+- **SLA Monitoring**: Track success rates and latency against targets
+
+### For Development Teams
+- **Local Monitoring**: Understand crawler behavior during development
+- **Performance Testing**: Measure impact of configuration changes
+- **Troubleshooting**: Quickly identify and fix issues
+- **Learning**: See exactly how the browser pool works
+
+## 🔄 Breaking Changes
+
+**None!** This release is fully backward compatible.
+
+- All existing Docker configurations continue to work
+- No API changes to existing endpoints
+- Monitoring is additive functionality
+- No migration required
+
+## 🚀 Upgrade Instructions
+
+### Docker
+
+```bash
+# Pull the latest version
+docker pull unclecode/crawl4ai:0.7.7
+
+# Or use the latest tag
+docker pull unclecode/crawl4ai:latest
+
+# Run with monitoring enabled (default)
+docker run -d \
+  -p 11235:11235 \
+  --shm-size=1g \
+  --name crawl4ai \
+  unclecode/crawl4ai:0.7.7
+
+# Access the monitoring dashboard
+open http://localhost:11235/dashboard
+```
+
+### Python Package
+
+```bash
+# Upgrade to latest version
+pip install --upgrade crawl4ai
+
+# Or install specific version
+pip install crawl4ai==0.7.7
+```
+
+## 🎬 Try the Demo
+
+Run the comprehensive demo that showcases all monitoring features:
+
+```bash
+python docs/releases_review/demo_v0.7.7.py
+```
+
+**The demo includes:**
+1. System health overview with live metrics
+2. Request tracking with active/completed monitoring
+3. Browser pool management (permanent/hot/cold)
+4. Complete Monitor API endpoint examples
+5. WebSocket streaming demonstration
+6. Control actions (cleanup, kill, restart)
+7. Production metrics and alerting patterns
+8. Self-hosting value proposition
+
+## 📚 Documentation
+
+### New Documentation
+- **[Self-Hosting Guide](https://docs.crawl4ai.com/core/self-hosting/)** - Complete self-hosting documentation with monitoring
+- **Demo Script**: `docs/releases_review/demo_v0.7.7.py` - Working examples
+
+### Updated Documentation
+- **Docker Deployment** → **Self-Hosting** (renamed for better positioning)
+- Added comprehensive monitoring sections
+- Production integration patterns
+- WebSocket streaming examples
+
+## 💡 Pro Tips
+
+1. **Start with the dashboard** - Visit `/dashboard` to get familiar with the monitoring system
+2. **Track the 6 key metrics** - Memory, success rate, latency, reuse rate, browser count, errors
+3. **Set up alerting early** - Use the Monitor API to build alerts before issues occur
+4. **Monitor browser pool efficiency** - Aim for >80% reuse rate for optimal performance
+5. **Use WebSocket for custom dashboards** - Build tailored monitoring UIs for your team
+6. **Leverage Prometheus integration** - Export metrics for long-term storage and analysis
+7. **Check janitor logs** - Understand automatic cleanup patterns
+8. **Use control actions judiciously** - Manual interventions are for exceptional cases
+
+## 🙏 Acknowledgments
+
+Thank you to our community for the feedback, bug reports, and feature requests that shaped this release. Special thanks to everyone who contributed to the issues that were fixed in this version.
+
+The monitoring system was built based on real user needs for production deployments, and your input made it comprehensive and practical.
+
+## 📞 Support & Resources
+
+- **📖 Documentation**: [docs.crawl4ai.com](https://docs.crawl4ai.com)
+- **🐙 GitHub**: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
+- **💬 Discord**: [discord.gg/crawl4ai](https://discord.gg/jP8KfhDhyN)
+- **🐦 Twitter**: [@unclecode](https://x.com/unclecode)
+- **📊 Dashboard**: `http://localhost:11235/dashboard` (when running)
+
+---
+
+**Crawl4AI v0.7.7 delivers complete self-hosting with enterprise-grade monitoring. You now have full visibility and control over your web crawling infrastructure. The monitoring dashboard, comprehensive API, and WebSocket streaming give you everything needed for production deployments. Try the self-hosting platform—it's a game changer for operational excellence!**
+
+**Happy crawling with full visibility!** 🕷️📊
+
+*- unclecode*

docs/md_v2/blog/releases/v0.7.8.md

@@ -0,0 +1,327 @@
+# Crawl4AI v0.7.8: Stability & Bug Fix Release
+
+*December 2025*
+
+---
+
+I'm releasing Crawl4AI v0.7.8—a focused stability release that addresses 11 bugs reported by the community. While there are no new features in this release, these fixes resolve important issues affecting Docker deployments, LLM extraction, URL handling, and dependency compatibility.
+
+## What's Fixed at a Glance
+
+- **Docker API**: Fixed ContentRelevanceFilter deserialization, ProxyConfig serialization, and cache folder permissions
+- **LLM Extraction**: Configurable rate limiter backoff, HTML input format support, and proper URL handling for raw HTML
+- **URL Handling**: Correct relative URL resolution after JavaScript redirects
+- **Dependencies**: Replaced deprecated PyPDF2 with pypdf, Pydantic v2 ConfigDict compatibility
+- **AdaptiveCrawler**: Fixed query expansion to actually use LLM instead of hardcoded mock data
+
+## Bug Fixes
+
+### Docker & API Fixes
+
+#### ContentRelevanceFilter Deserialization (#1642)
+
+**The Problem:** When sending deep crawl requests to the Docker API with `ContentRelevanceFilter`, the server failed to deserialize the filter, causing requests to fail.
+
+**The Fix:** I added `ContentRelevanceFilter` to the public exports and enhanced the deserialization logic with dynamic imports.
+
+```python
+# This now works correctly in Docker API
+import httpx
+
+request = {
+    "urls": ["https://docs.example.com"],
+    "crawler_config": {
+        "deep_crawl_strategy": {
+            "type": "BFSDeepCrawlStrategy",
+            "max_depth": 2,
+            "filter_chain": [
+                {
+                    "type": "ContentRelevanceFilter",
+                    "query": "API documentation",
+                    "threshold": 0.3
+                }
+            ]
+        }
+    }
+}
+
+async with httpx.AsyncClient() as client:
+    response = await client.post("http://localhost:11235/crawl", json=request)
+    # Previously failed, now works!
+```
+
+#### ProxyConfig JSON Serialization (#1629)
+
+**The Problem:** `BrowserConfig.to_dict()` failed when `proxy_config` was set because `ProxyConfig` wasn't being serialized to a dictionary.
+
+**The Fix:** `ProxyConfig.to_dict()` is now called during serialization.
+
+```python
+from crawl4ai import BrowserConfig
+from crawl4ai.async_configs import ProxyConfig
+
+proxy = ProxyConfig(
+    server="http://proxy.example.com:8080",
+    username="user",
+    password="pass"
+)
+
+config = BrowserConfig(headless=True, proxy_config=proxy)
+
+# Previously raised TypeError, now works
+config_dict = config.to_dict()
+json.dumps(config_dict)  # Valid JSON
+```
+
+#### Docker Cache Folder Permissions (#1638)
+
+**The Problem:** The `.cache` folder in the Docker image had incorrect permissions, causing crawling to fail when caching was enabled.
+
+**The Fix:** Corrected ownership and permissions during image build.
+
+```bash
+# Cache now works correctly in Docker
+docker run -d -p 11235:11235 \
+    --shm-size=1g \
+    -v ./my-cache:/app/.cache \
+    unclecode/crawl4ai:0.7.8
+```
+
+---
+
+### LLM & Extraction Fixes
+
+#### Configurable Rate Limiter Backoff (#1269)
+
+**The Problem:** The LLM rate limiting backoff parameters were hardcoded, making it impossible to adjust retry behavior for different API rate limits.
+
+**The Fix:** `LLMConfig` now accepts three new parameters for complete control over retry behavior.
+
+```python
+from crawl4ai import LLMConfig
+
+# Default behavior (unchanged)
+default_config = LLMConfig(provider="openai/gpt-4o-mini")
+# backoff_base_delay=2, backoff_max_attempts=3, backoff_exponential_factor=2
+
+# Custom configuration for APIs with strict rate limits
+custom_config = LLMConfig(
+    provider="openai/gpt-4o-mini",
+    backoff_base_delay=5,           # Wait 5 seconds on first retry
+    backoff_max_attempts=5,          # Try up to 5 times
+    backoff_exponential_factor=3     # Multiply delay by 3 each attempt
+)
+
+# Retry sequence: 5s -> 15s -> 45s -> 135s -> 405s
+```
+
+#### LLM Strategy HTML Input Support (#1178)
+
+**The Problem:** `LLMExtractionStrategy` always sent markdown to the LLM, but some extraction tasks work better with HTML structure preserved.
+
+**The Fix:** Added `input_format` parameter supporting `"markdown"`, `"html"`, `"fit_markdown"`, `"cleaned_html"`, and `"fit_html"`.
+
+```python
+from crawl4ai import LLMExtractionStrategy, LLMConfig
+
+# Default: markdown input (unchanged)
+markdown_strategy = LLMExtractionStrategy(
+    llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+    instruction="Extract product information"
+)
+
+# NEW: HTML input - preserves table/list structure
+html_strategy = LLMExtractionStrategy(
+    llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+    instruction="Extract the data table preserving structure",
+    input_format="html"
+)
+
+# NEW: Filtered markdown - only relevant content
+fit_strategy = LLMExtractionStrategy(
+    llm_config=LLMConfig(provider="openai/gpt-4o-mini"),
+    instruction="Summarize the main content",
+    input_format="fit_markdown"
+)
+```
+
+#### Raw HTML URL Variable (#1116)
+
+**The Problem:** When using `url="raw:<html>..."`, the entire HTML content was being passed to extraction strategies as the URL parameter, polluting LLM prompts.
+
+**The Fix:** The URL is now correctly set to `"Raw HTML"` for raw HTML inputs.
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+html = "<html><body><h1>Test</h1></body></html>"
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(
+        url=f"raw:{html}",
+        config=CrawlerRunConfig(extraction_strategy=my_strategy)
+    )
+    # extraction_strategy receives url="Raw HTML" instead of the HTML blob
+```
+
+---
+
+### URL Handling Fix
+
+#### Relative URLs After Redirects (#1268)
+
+**The Problem:** When JavaScript caused a page redirect, relative links were resolved against the original URL instead of the final URL.
+
+**The Fix:** `redirected_url` now captures the actual page URL after all JavaScript execution completes.
+
+```python
+from crawl4ai import AsyncWebCrawler
+
+async with AsyncWebCrawler() as crawler:
+    # Page at /old-page redirects via JS to /new-page
+    result = await crawler.arun(url="https://example.com/old-page")
+
+    # BEFORE: redirected_url = "https://example.com/old-page"
+    # AFTER:  redirected_url = "https://example.com/new-page"
+
+    # Links are now correctly resolved against the final URL
+    for link in result.links['internal']:
+        print(link['href'])  # Relative links resolved correctly
+```
+
+---
+
+### Dependency & Compatibility Fixes
+
+#### PyPDF2 Replaced with pypdf (#1412)
+
+**The Problem:** PyPDF2 was deprecated in 2022 and is no longer maintained.
+
+**The Fix:** Replaced with the actively maintained `pypdf` library.
+
+```python
+# Installation (unchanged)
+pip install crawl4ai[pdf]
+
+# The PDF processor now uses pypdf internally
+# No code changes required - API remains the same
+```
+
+#### Pydantic v2 ConfigDict Compatibility (#678)
+
+**The Problem:** Using the deprecated `class Config` syntax caused deprecation warnings with Pydantic v2.
+
+**The Fix:** Migrated to `model_config = ConfigDict(...)` syntax.
+
+```python
+# No more deprecation warnings when importing crawl4ai models
+from crawl4ai.models import CrawlResult
+from crawl4ai import CrawlerRunConfig, BrowserConfig
+
+# All models are now Pydantic v2 compatible
+```
+
+---
+
+### AdaptiveCrawler Fix
+
+#### Query Expansion Using LLM (#1621)
+
+**The Problem:** The `EmbeddingStrategy` in AdaptiveCrawler had commented-out LLM code and was using hardcoded mock query variations instead.
+
+**The Fix:** Uncommented and activated the LLM call for actual query expansion.
+
+```python
+# AdaptiveCrawler query expansion now actually uses the LLM
+# Instead of hardcoded variations like:
+# variations = {'queries': ['what are the best vegetables...']}
+
+# The LLM generates relevant query variations based on your actual query
+```
+
+---
+
+### Code Formatting Fix
+
+#### Import Statement Formatting (#1181)
+
+**The Problem:** When extracting code from web pages, import statements were sometimes concatenated without proper line separation.
+
+**The Fix:** Import statements now maintain proper newline separation.
+
+```python
+# BEFORE: "import osimport sysfrom pathlib import Path"
+# AFTER:
+# import os
+# import sys
+# from pathlib import Path
+```
+
+---
+
+## Breaking Changes
+
+**None!** This release is fully backward compatible.
+
+- All existing code continues to work without modification
+- New parameters have sensible defaults matching previous behavior
+- No API changes to existing functionality
+
+---
+
+## Upgrade Instructions
+
+### Python Package
+
+```bash
+pip install --upgrade crawl4ai
+# or
+pip install crawl4ai==0.7.8
+```
+
+### Docker
+
+```bash
+# Pull the latest version
+docker pull unclecode/crawl4ai:0.7.8
+
+# Run
+docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.8
+```
+
+---
+
+## Verification
+
+Run the verification tests to confirm all fixes are working:
+
+```bash
+python docs/releases_review/demo_v0.7.8.py
+```
+
+This runs actual tests that verify each bug fix is properly implemented.
+
+---
+
+## Acknowledgments
+
+Thank you to everyone who reported these issues and provided detailed reproduction steps. Your bug reports make Crawl4AI better for everyone.
+
+Issues fixed: #1642, #1638, #1629, #1621, #1412, #1269, #1268, #1181, #1178, #1116, #678
+
+---
+
+## Support & Resources
+
+- **Documentation**: [docs.crawl4ai.com](https://docs.crawl4ai.com)
+- **GitHub**: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
+- **Discord**: [discord.gg/crawl4ai](https://discord.gg/jP8KfhDhyN)
+- **Twitter**: [@unclecode](https://x.com/unclecode)
+
+---
+
+**This stability release ensures Crawl4AI works reliably across Docker deployments, LLM extraction workflows, and various edge cases. Thank you for your continued support and feedback!**
+
+**Happy crawling!**
+
+*- unclecode*

docs/md_v2/blog/releases/v0.8.0.md

@@ -0,0 +1,243 @@
+# Crawl4AI v0.8.0 Release Notes
+
+**Release Date**: January 2026
+**Previous Version**: v0.7.6
+**Status**: Release Candidate
+
+---
+
+## Highlights
+
+- **Critical Security Fixes** for Docker API deployment
+- **11 New Features** including crash recovery, prefetch mode, and proxy improvements
+- **Breaking Changes** - see migration guide below
+
+---
+
+## Breaking Changes
+
+### 1. Docker API: Hooks Disabled by Default
+
+**What changed**: Hooks are now disabled by default on the Docker API.
+
+**Why**: Security fix for Remote Code Execution (RCE) vulnerability.
+
+**Who is affected**: Users of the Docker API who use the `hooks` parameter in `/crawl` requests.
+
+**Migration**:
+```bash
+# To re-enable hooks (only if you trust all API users):
+export CRAWL4AI_HOOKS_ENABLED=true
+```
+
+### 2. Docker API: file:// URLs Blocked
+
+**What changed**: The endpoints `/execute_js`, `/screenshot`, `/pdf`, and `/html` now reject `file://` URLs.
+
+**Why**: Security fix for Local File Inclusion (LFI) vulnerability.
+
+**Who is affected**: Users who were reading local files via the Docker API.
+
+**Migration**: Use the Python library directly for local file processing:
+```python
+# Instead of API call with file:// URL, use library:
+from crawl4ai import AsyncWebCrawler
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun(url="file:///path/to/file.html")
+```
+
+---
+
+## Security Fixes
+
+### Critical: Remote Code Execution via Hooks (CVE Pending)
+
+**Severity**: CRITICAL (CVSS 10.0)
+**Affected**: Docker API deployment (all versions before v0.8.0)
+**Vector**: `POST /crawl` with malicious `hooks` parameter
+
+**Details**: The `__import__` builtin was available in hook code, allowing attackers to import `os`, `subprocess`, etc. and execute arbitrary commands.
+
+**Fix**:
+1. Removed `__import__` from allowed builtins
+2. Hooks disabled by default (`CRAWL4AI_HOOKS_ENABLED=false`)
+
+### High: Local File Inclusion via file:// URLs (CVE Pending)
+
+**Severity**: HIGH (CVSS 8.6)
+**Affected**: Docker API deployment (all versions before v0.8.0)
+**Vector**: `POST /execute_js` (and other endpoints) with `file:///etc/passwd`
+
+**Details**: API endpoints accepted `file://` URLs, allowing attackers to read arbitrary files from the server.
+
+**Fix**: URL scheme validation now only allows `http://`, `https://`, and `raw:` URLs.
+
+### Credits
+
+Discovered by **Neo by ProjectDiscovery** ([projectdiscovery.io](https://projectdiscovery.io)) - December 2025
+
+---
+
+## New Features
+
+### 1. init_scripts Support for BrowserConfig
+
+Pre-page-load JavaScript injection for stealth evasions.
+
+```python
+config = BrowserConfig(
+    init_scripts=[
+        "Object.defineProperty(navigator, 'webdriver', {get: () => false})"
+    ]
+)
+```
+
+### 2. CDP Connection Improvements
+
+- WebSocket URL support (`ws://`, `wss://`)
+- Proper cleanup with `cdp_cleanup_on_close=True`
+- Browser reuse across multiple connections
+
+### 3. Crash Recovery for Deep Crawl Strategies
+
+All deep crawl strategies (BFS, DFS, Best-First) now support crash recovery:
+
+```python
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+strategy = BFSDeepCrawlStrategy(
+    max_depth=3,
+    resume_state=saved_state,  # Resume from checkpoint
+    on_state_change=save_callback  # Persist state in real-time
+)
+```
+
+### 4. PDF and MHTML for raw:/file:// URLs
+
+Generate PDFs and MHTML from cached HTML content.
+
+### 5. Screenshots for raw:/file:// URLs
+
+Render cached HTML and capture screenshots.
+
+### 6. base_url Parameter for CrawlerRunConfig
+
+Proper URL resolution for raw: HTML processing:
+
+```python
+config = CrawlerRunConfig(base_url='https://example.com')
+result = await crawler.arun(url='raw:{html}', config=config)
+```
+
+### 7. Prefetch Mode for Two-Phase Deep Crawling
+
+Fast link extraction without full page processing:
+
+```python
+config = CrawlerRunConfig(prefetch=True)
+```
+
+### 8. Proxy Rotation and Configuration
+
+Enhanced proxy rotation with sticky sessions support.
+
+### 9. Proxy Support for HTTP Strategy
+
+Non-browser crawler now supports proxies.
+
+### 10. Browser Pipeline for raw:/file:// URLs
+
+New `process_in_browser` parameter for browser operations on local content:
+
+```python
+config = CrawlerRunConfig(
+    process_in_browser=True,  # Force browser processing
+    screenshot=True
+)
+result = await crawler.arun(url='raw:<html>...</html>', config=config)
+```
+
+### 11. Smart TTL Cache for Sitemap URL Seeder
+
+Intelligent cache invalidation for sitemaps:
+
+```python
+config = SeedingConfig(
+    cache_ttl_hours=24,
+    validate_sitemap_lastmod=True
+)
+```
+
+---
+
+## Bug Fixes
+
+### raw: URL Parsing Truncates at # Character
+
+**Problem**: CSS color codes like `#eee` were being truncated.
+
+**Before**: `raw:body{background:#eee}` → `body{background:`
+**After**: `raw:body{background:#eee}` → `body{background:#eee}`
+
+### Caching System Improvements
+
+Various fixes to cache validation and persistence.
+
+---
+
+## Documentation Updates
+
+- Multi-sample schema generation documentation
+- URL seeder smart TTL cache parameters
+- Security documentation (SECURITY.md)
+
+---
+
+## Upgrade Guide
+
+### From v0.7.x to v0.8.0
+
+1. **Update the package**:
+   ```bash
+   pip install --upgrade crawl4ai
+   ```
+
+2. **Docker API users**:
+   - Hooks are now disabled by default
+   - If you need hooks: `export CRAWL4AI_HOOKS_ENABLED=true`
+   - `file://` URLs no longer work on API (use library directly)
+
+3. **Review security settings**:
+   ```yaml
+   # config.yml - recommended for production
+   security:
+     enabled: true
+     jwt_enabled: true
+   ```
+
+4. **Test your integration** before deploying to production
+
+### Breaking Change Checklist
+
+- [ ] Check if you use `hooks` parameter in API calls
+- [ ] Check if you use `file://` URLs via the API
+- [ ] Update environment variables if needed
+- [ ] Review security configuration
+
+---
+
+## Full Changelog
+
+See [CHANGELOG.md](../CHANGELOG.md) for complete version history.
+
+---
+
+## Contributors
+
+Thanks to all contributors who made this release possible.
+
+Special thanks to **Neo by ProjectDiscovery** for responsible security disclosure.
+
+---
+
+*For questions or issues, please open a [GitHub Issue](https://github.com/unclecode/crawl4ai/issues).*

docs/md_v2/core/browser-crawler-config.md

@@ -17,6 +17,11 @@ class BrowserConfig:
     def __init__(
         browser_type="chromium",
         headless=True,
+        browser_mode="dedicated",
+        use_managed_browser=False,
+        cdp_url=None,
+        debugging_port=9222,
+        host="localhost",
         proxy_config=None,
         viewport_width=1080,
         viewport_height=600,
@@ -25,7 +30,13 @@ class BrowserConfig:
         user_data_dir=None,
         cookies=None,
         headers=None,
-        user_agent=None,
+        user_agent=(
+            # "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) AppleWebKit/537.36 "
+            # "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+            # "(KHTML, like Gecko) Chrome/116.0.5845.187 Safari/604.1 Edg/117.0.2045.47"
+            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36"
+        ),
+        user_agent_mode="",
         text_mode=False,
         light_mode=False,
         extra_args=None,
@@ -37,17 +48,33 @@ class BrowserConfig:
 
 ### Key Fields to Note
 
-1. **`browser_type`**  
-- Options: `"chromium"`, `"firefox"`, or `"webkit"`.  
-- Defaults to `"chromium"`.  
-- If you need a different engine, specify it here.
+1.⠀**`browser_type`**  
+   - Options: `"chromium"`, `"firefox"`, or `"webkit"`.  
+   - Defaults to `"chromium"`.  
+   - If you need a different engine, specify it here.
 
-2. **`headless`**  
+2.⠀**`headless`**  
    - `True`: Runs the browser in headless mode (invisible browser).  
    - `False`: Runs the browser in visible mode, which helps with debugging.
 
-3. **`proxy_config`**  
-   - A dictionary with fields like:  
+3.⠀**`browser_mode`**  
+   - Determines how the browser should be initialized:
+     - `"dedicated"` (default): Creates a new browser instance each time
+     - `"builtin"`: Uses the builtin CDP browser running in background
+     - `"custom"`: Uses explicit CDP settings provided in `cdp_url`
+     - `"docker"`: Runs browser in Docker container with isolation
+
+4.⠀**`use_managed_browser`** & **`cdp_url`**  
+   - `use_managed_browser=True`: Launch browser using Chrome DevTools Protocol (CDP) for advanced control
+   - `cdp_url`: URL for CDP endpoint (e.g., `"ws://localhost:9222/devtools/browser/"`)
+   - Automatically set based on `browser_mode`
+
+5.⠀**`debugging_port`** & **`host`**  
+   - `debugging_port`: Port for browser debugging protocol (default: 9222)
+   - `host`: Host for browser connection (default: "localhost")
+
+6.⠀**`proxy_config`**  
+   - A `ProxyConfig` object or dictionary with fields like:  
 ```json
 {
     "server": "http://proxy.example.com:8080", 
@@ -57,35 +84,35 @@ class BrowserConfig:
 ```
    - Leave as `None` if a proxy is not required.
 
-4. **`viewport_width` & `viewport_height`**:  
+7.⠀**`viewport_width` & `viewport_height`**  
    - The initial window size.  
    - Some sites behave differently with smaller or bigger viewports.
 
-5. **`verbose`**:  
+8.⠀**`verbose`**  
    - If `True`, prints extra logs.  
    - Handy for debugging.
 
-6. **`use_persistent_context`**:  
+9.⠀**`use_persistent_context`**  
    - If `True`, uses a **persistent** browser profile, storing cookies/local storage across runs.  
    - Typically also set `user_data_dir` to point to a folder.
 
-7. **`cookies`** & **`headers`**:  
-   - If you want to start with specific cookies or add universal HTTP headers, set them here.  
-   - E.g. `cookies=[{"name": "session", "value": "abc123", "domain": "example.com"}]`.
+10.⠀**`cookies`** & **`headers`**  
+    - If you want to start with specific cookies or add universal HTTP headers to the browser context, set them here.  
+    - E.g. `cookies=[{"name": "session", "value": "abc123", "domain": "example.com"}]`.
 
-8. **`user_agent`**:  
-   - Custom User-Agent string. If `None`, a default is used.  
-   - You can also set `user_agent_mode="random"` for randomization (if you want to fight bot detection).
+11.⠀**`user_agent`** & **`user_agent_mode`**  
+    - `user_agent`: Custom User-Agent string. If `None`, a default is used.  
+    - `user_agent_mode`: Set to `"random"` for randomization (helps fight bot detection).
 
-9. **`text_mode`** & **`light_mode`**:  
-   - `text_mode=True` disables images, possibly speeding up text-only crawls.  
-   - `light_mode=True` turns off certain background features for performance.  
+12.⠀**`text_mode`** & **`light_mode`**  
+    - `text_mode=True` disables images, possibly speeding up text-only crawls.  
+    - `light_mode=True` turns off certain background features for performance.  
 
-10. **`extra_args`**:  
+13.⠀**`extra_args`**  
     - Additional flags for the underlying browser.  
     - E.g. `["--disable-extensions"]`.
 
-11. **`enable_stealth`**:  
+14.⠀**`enable_stealth`**  
     - If `True`, enables stealth mode using playwright-stealth.  
     - Modifies browser fingerprints to avoid basic bot detection.  
     - Default is `False`. Recommended for sites with bot protection.
@@ -134,9 +161,11 @@ class CrawlerRunConfig:
     def __init__(
         word_count_threshold=200,
         extraction_strategy=None,
+        chunking_strategy=RegexChunking(),
         markdown_generator=None,
-        cache_mode=None,
+        cache_mode=CacheMode.BYPASS,
         js_code=None,
+        c4a_script=None,
         wait_for=None,
         screenshot=False,
         pdf=False,
@@ -145,13 +174,18 @@ class CrawlerRunConfig:
         locale=None,            # e.g. "en-US", "fr-FR"
         timezone_id=None,       # e.g. "America/New_York"
         geolocation=None,       # GeolocationConfig object
-        # Resource Management
-        enable_rate_limiting=False,
-        rate_limit_config=None,
-        memory_threshold_percent=70.0,
-        check_interval=1.0,
-        max_session_permit=20,
-        display_mode=None,
+        # Proxy Configuration
+        proxy_config=None,
+        proxy_rotation_strategy=None,
+        # Page Interaction Parameters
+        scan_full_page=False,
+        scroll_delay=0.2,
+        wait_until="domcontentloaded",
+        page_timeout=60000,
+        delay_before_return_html=0.1,
+        # URL Matching Parameters
+        url_matcher=None,       # For URL-specific configurations
+        match_mode=MatchMode.OR,
         verbose=True,
         stream=False,  # Enable streaming for arun_many()
         # ... other advanced parameters omitted
@@ -161,69 +195,68 @@ class CrawlerRunConfig:
 
 ### Key Fields to Note
 
-1. **`word_count_threshold`**:  
+1.⠀**`word_count_threshold`**:  
    - The minimum word count before a block is considered.  
    - If your site has lots of short paragraphs or items, you can lower it.
 
-2. **`extraction_strategy`**:  
+2.⠀**`extraction_strategy`**:  
    - Where you plug in JSON-based extraction (CSS, LLM, etc.).  
    - If `None`, no structured extraction is done (only raw/cleaned HTML + markdown).
 
-3. **`markdown_generator`**:  
+3.⠀**`chunking_strategy`**:  
+   - Strategy to chunk content before extraction.  
+   - Defaults to `RegexChunking()`. Can be customized for different chunking approaches.
+
+4.⠀**`markdown_generator`**:  
    - E.g., `DefaultMarkdownGenerator(...)`, controlling how HTML→Markdown conversion is done.  
    - If `None`, a default approach is used.
 
-4. **`cache_mode`**:  
+5.⠀**`cache_mode`**:  
    - Controls caching behavior (`ENABLED`, `BYPASS`, `DISABLED`, etc.).  
-   - If `None`, defaults to some level of caching or you can specify `CacheMode.ENABLED`.
+   - Defaults to `CacheMode.BYPASS`.
 
-5. **`js_code`**:  
-   - A string or list of JS strings to execute.  
+6.⠀**`js_code`** & **`c4a_script`**:  
+   - `js_code`: A string or list of JavaScript strings to execute.  
+   - `c4a_script`: C4A script that compiles to JavaScript.
    - Great for "Load More" buttons or user interactions.  
 
-6. **`wait_for`**:  
+7.⠀**`wait_for`**:  
    - A CSS or JS expression to wait for before extracting content.  
    - Common usage: `wait_for="css:.main-loaded"` or `wait_for="js:() => window.loaded === true"`.
 
-7. **`screenshot`**, **`pdf`**, & **`capture_mhtml`**:  
+8.⠀**`screenshot`**, **`pdf`**, & **`capture_mhtml`**:  
    - If `True`, captures a screenshot, PDF, or MHTML snapshot after the page is fully loaded.  
    - The results go to `result.screenshot` (base64), `result.pdf` (bytes), or `result.mhtml` (string).
 
-8. **Location Parameters**:  
+9.⠀**Location Parameters**:  
    - **`locale`**: Browser's locale (e.g., `"en-US"`, `"fr-FR"`) for language preferences
    - **`timezone_id`**: Browser's timezone (e.g., `"America/New_York"`, `"Europe/Paris"`)
    - **`geolocation`**: GPS coordinates via `GeolocationConfig(latitude=48.8566, longitude=2.3522)`
    - See [Identity Based Crawling](../advanced/identity-based-crawling.md#7-locale-timezone-and-geolocation-control)
 
-9. **`verbose`**:  
-   - Logs additional runtime details.  
-   - Overlaps with the browser's verbosity if also set to `True` in `BrowserConfig`.
+10.⠀**Proxy Configuration**:  
+    - **`proxy_config`**: Proxy server configuration (ProxyConfig object or dict) e.g. {"server": "...", "username": "...", "password"}
+    - **`proxy_rotation_strategy`**: Strategy for rotating proxies during crawls
 
-10. **`enable_rate_limiting`**:  
-   - If `True`, enables rate limiting for batch processing.  
-   - Requires `rate_limit_config` to be set.
+11.⠀**Page Interaction Parameters**:  
+    - **`scan_full_page`**: If `True`, scroll through the entire page to load all content
+    - **`wait_until`**: Condition to wait for when navigating (e.g., "domcontentloaded", "networkidle")
+    - **`page_timeout`**: Timeout in milliseconds for page operations (default: 60000)
+    - **`delay_before_return_html`**: Delay in seconds before retrieving final HTML.
 
-11. **`memory_threshold_percent`**:  
-    - The memory threshold (as a percentage) to monitor.  
-    - If exceeded, the crawler will pause or slow down.
-
-12. **`check_interval`**:  
-    - The interval (in seconds) to check system resources.  
-    - Affects how often memory and CPU usage are monitored.
-
-13. **`max_session_permit`**:  
-    - The maximum number of concurrent crawl sessions.  
-    - Helps prevent overwhelming the system.
-
-14. **`url_matcher`** & **`match_mode`**:  
+12.⠀**`url_matcher`** & **`match_mode`**:  
     - Enable URL-specific configurations when used with `arun_many()`.
     - Set `url_matcher` to patterns (glob, function, or list) to match specific URLs.
     - Use `match_mode` (OR/AND) to control how multiple patterns combine.
     - See [URL-Specific Configurations](../api/arun_many.md#url-specific-configurations) for examples.
 
-15. **`display_mode`**:  
-    - The display mode for progress information (`DETAILED`, `BRIEF`, etc.).  
-    - Affects how much information is printed during the crawl.
+13.⠀**`verbose`**:  
+    - Logs additional runtime details.  
+    - Overlaps with the browser's verbosity if also set to `True` in `BrowserConfig`.
+
+14.⠀**`stream`**:  
+    - If `True`, enables streaming mode for `arun_many()` to process URLs as they complete.
+    - Allows handling results incrementally instead of waiting for all URLs to finish.
 
 
 ### Helper Methods
@@ -263,20 +296,32 @@ The `clone()` method:
 
 ### Key fields to note
 
-1. **`provider`**:  
+1.⠀**`provider`**:  
 - Which LLM provider to use. 
 - Possible values are `"ollama/llama3","groq/llama3-70b-8192","groq/llama3-8b-8192", "openai/gpt-4o-mini" ,"openai/gpt-4o","openai/o1-mini","openai/o1-preview","openai/o3-mini","openai/o3-mini-high","anthropic/claude-3-haiku-20240307","anthropic/claude-3-opus-20240229","anthropic/claude-3-sonnet-20240229","anthropic/claude-3-5-sonnet-20240620","gemini/gemini-pro","gemini/gemini-1.5-pro","gemini/gemini-2.0-flash","gemini/gemini-2.0-flash-exp","gemini/gemini-2.0-flash-lite-preview-02-05","deepseek/deepseek-chat"`<br/>*(default: `"openai/gpt-4o-mini"`)*
 
-2. **`api_token`**:  
+2.⠀**`api_token`**:  
     - Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`"GEMINI_API_KEY"` will be read from environment variables  
     - API token of LLM provider <br/> eg: `api_token = "gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv"`
     - Environment variable - use with prefix "env:" <br/> eg:`api_token = "env: GROQ_API_KEY"`            
 
-3. **`base_url`**:  
+3.⠀**`base_url`**:  
    - If your provider has a custom endpoint
 
+4.⠀**Retry/backoff controls** *(optional)*:  
+   - `backoff_base_delay` *(default `2` seconds)* – base delay inserted before the first retry when the provider returns a rate-limit response.  
+   - `backoff_max_attempts` *(default `3`)* – total number of attempts (initial call plus retries) before the request is surfaced as an error.  
+   - `backoff_exponential_factor` *(default `2`)* – growth rate for the retry delay (`delay = base_delay * factor^attempt`).  
+   - These values are forwarded to the shared `perform_completion_with_backoff` helper, ensuring every strategy that consumes your `LLMConfig` honors the same throttling policy.
+
 ```python
-llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
+llm_config = LLMConfig(
+    provider="openai/gpt-4o-mini",
+    api_token=os.getenv("OPENAI_API_KEY"),
+    backoff_base_delay=1, # optional
+    backoff_max_attempts=5, # optional
+    backoff_exponential_factor=3, #optional
+)
 ```
 
 ## 4. Putting It All Together

docs/md_v2/core/c4a-script.md

@@ -69,12 +69,12 @@ The tutorial includes a Flask-based web interface with:
 cd docs/examples/c4a_script/tutorial/
 
 # Install dependencies
-pip install flask
+pip install -r requirements.txt
 
 # Launch the tutorial server
-python app.py
+python server.py
 
-# Open http://localhost:5000 in your browser
+# Open http://localhost:8000 in your browser
 ```
 
 ## Core Concepts
@@ -111,8 +111,8 @@ CLICK `.submit-btn`
 # By attribute
 CLICK `button[type="submit"]`
 
-# By text content
-CLICK `button:contains("Sign In")`
+# By accessible attributes
+CLICK `button[aria-label="Search"][title="Search"]`
 
 # Complex selectors
 CLICK `.form-container input[name="email"]`

docs/md_v2/core/deep-crawling.md

@@ -4,11 +4,13 @@ One of Crawl4AI's most powerful features is its ability to perform **configurabl
 
 In this tutorial, you'll learn:
 
-1. How to set up a **Basic Deep Crawler** with BFS strategy  
-2. Understanding the difference between **streamed and non-streamed** output  
-3. Implementing **filters and scorers** to target specific content  
-4. Creating **advanced filtering chains** for sophisticated crawls  
-5. Using **BestFirstCrawling** for intelligent exploration prioritization  
+1. How to set up a **Basic Deep Crawler** with BFS strategy
+2. Understanding the difference between **streamed and non-streamed** output
+3. Implementing **filters and scorers** to target specific content
+4. Creating **advanced filtering chains** for sophisticated crawls
+5. Using **BestFirstCrawling** for intelligent exploration prioritization
+6. **Crash recovery** for long-running production crawls
+7. **Prefetch mode** for fast URL discovery  
 
 > **Prerequisites**  
 > - You’ve completed or read [AsyncWebCrawler Basics](../core/simple-crawling.md) to understand how to run a simple crawl.  
@@ -485,7 +487,249 @@ This is especially useful for security-conscious crawling or when dealing with s
 
 ---
 
-## 10. Summary & Next Steps
+## 10. Crash Recovery for Long-Running Crawls
+
+For production deployments, especially in cloud environments where instances can be terminated unexpectedly, Crawl4AI provides built-in crash recovery support for all deep crawl strategies.
+
+### 10.1 Enabling State Persistence
+
+All deep crawl strategies (BFS, DFS, Best-First) support two optional parameters:
+
+- **`resume_state`**: Pass a previously saved state to resume from a checkpoint
+- **`on_state_change`**: Async callback fired after each URL is processed
+
+```python
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+import json
+
+# Callback to save state after each URL
+async def save_state_to_redis(state: dict):
+    await redis.set("crawl_state", json.dumps(state))
+
+strategy = BFSDeepCrawlStrategy(
+    max_depth=3,
+    on_state_change=save_state_to_redis,  # Called after each URL
+)
+```
+
+### 10.2 State Structure
+
+The state dictionary is JSON-serializable and contains:
+
+```python
+{
+    "strategy_type": "bfs",  # or "dfs", "best_first"
+    "visited": ["url1", "url2", ...],  # Already crawled URLs
+    "pending": [{"url": "...", "parent_url": "..."}],  # Queue/stack
+    "depths": {"url1": 0, "url2": 1},  # Depth tracking
+    "pages_crawled": 42  # Counter
+}
+```
+
+### 10.3 Resuming from a Checkpoint
+
+```python
+import json
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+# Load saved state (e.g., from Redis, database, or file)
+saved_state = json.loads(await redis.get("crawl_state"))
+
+# Resume crawling from where we left off
+strategy = BFSDeepCrawlStrategy(
+    max_depth=3,
+    resume_state=saved_state,  # Continue from checkpoint
+    on_state_change=save_state_to_redis,  # Keep saving progress
+)
+
+config = CrawlerRunConfig(deep_crawl_strategy=strategy)
+
+async with AsyncWebCrawler() as crawler:
+    # Will skip already-visited URLs and continue from pending queue
+    results = await crawler.arun(start_url, config=config)
+```
+
+### 10.4 Manual State Export
+
+You can export the last captured state using `export_state()`. Note that this requires `on_state_change` to be set (state is captured in the callback):
+
+```python
+import json
+
+captured_state = None
+
+async def capture_state(state: dict):
+    global captured_state
+    captured_state = state
+
+strategy = BFSDeepCrawlStrategy(
+    max_depth=2,
+    on_state_change=capture_state,  # Required for state capture
+)
+config = CrawlerRunConfig(deep_crawl_strategy=strategy)
+
+async with AsyncWebCrawler() as crawler:
+    results = await crawler.arun(start_url, config=config)
+
+# Get the last captured state
+state = strategy.export_state()
+if state:
+    # Save to your preferred storage
+    with open("crawl_checkpoint.json", "w") as f:
+        json.dump(state, f)
+```
+
+### 10.5 Complete Example: Redis-Based Recovery
+
+```python
+import asyncio
+import json
+import redis.asyncio as redis
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+
+REDIS_KEY = "crawl4ai:crawl_state"
+
+async def main():
+    redis_client = redis.Redis(host='localhost', port=6379, db=0)
+
+    # Check for existing state
+    saved_state = None
+    existing = await redis_client.get(REDIS_KEY)
+    if existing:
+        saved_state = json.loads(existing)
+        print(f"Resuming from checkpoint: {saved_state['pages_crawled']} pages already crawled")
+
+    # State persistence callback
+    async def persist_state(state: dict):
+        await redis_client.set(REDIS_KEY, json.dumps(state))
+
+    # Create strategy with recovery support
+    strategy = BFSDeepCrawlStrategy(
+        max_depth=3,
+        max_pages=100,
+        resume_state=saved_state,
+        on_state_change=persist_state,
+    )
+
+    config = CrawlerRunConfig(deep_crawl_strategy=strategy, stream=True)
+
+    try:
+        async with AsyncWebCrawler() as crawler:
+            async for result in await crawler.arun("https://example.com", config=config):
+                print(f"Crawled: {result.url}")
+    except Exception as e:
+        print(f"Crawl interrupted: {e}")
+        print("State saved - restart to resume")
+    finally:
+        await redis_client.close()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### 10.6 Zero Overhead
+
+When `resume_state=None` and `on_state_change=None` (the defaults), there is no performance impact. State tracking only activates when you enable these features.
+
+---
+
+## 11. Prefetch Mode for Fast URL Discovery
+
+When you need to quickly discover URLs without full page processing, use **prefetch mode**. This is ideal for two-phase crawling where you first map the site, then selectively process specific pages.
+
+### 11.1 Enabling Prefetch Mode
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+config = CrawlerRunConfig(prefetch=True)
+
+async with AsyncWebCrawler() as crawler:
+    result = await crawler.arun("https://example.com", config=config)
+
+    # Result contains only HTML and links - no markdown, no extraction
+    print(f"Found {len(result.links['internal'])} internal links")
+    print(f"Found {len(result.links['external'])} external links")
+```
+
+### 11.2 What Gets Skipped
+
+Prefetch mode uses a fast path that bypasses heavy processing:
+
+| Processing Step | Normal Mode | Prefetch Mode |
+|----------------|-------------|---------------|
+| Fetch HTML | ✅ | ✅ |
+| Extract links | ✅ | ✅ (fast `quick_extract_links()`) |
+| Generate markdown | ✅ | ❌ Skipped |
+| Content scraping | ✅ | ❌ Skipped |
+| Media extraction | ✅ | ❌ Skipped |
+| LLM extraction | ✅ | ❌ Skipped |
+
+### 11.3 Performance Benefit
+
+- **Normal mode**: Full pipeline (~2-5 seconds per page)
+- **Prefetch mode**: HTML + links only (~200-500ms per page)
+
+This makes prefetch mode **5-10x faster** for URL discovery.
+
+### 11.4 Two-Phase Crawling Pattern
+
+The most common use case is two-phase crawling:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def two_phase_crawl(start_url: str):
+    async with AsyncWebCrawler() as crawler:
+        # ═══════════════════════════════════════════════
+        # Phase 1: Fast discovery (prefetch mode)
+        # ═══════════════════════════════════════════════
+        prefetch_config = CrawlerRunConfig(prefetch=True)
+        discovery = await crawler.arun(start_url, config=prefetch_config)
+
+        all_urls = [link["href"] for link in discovery.links.get("internal", [])]
+        print(f"Discovered {len(all_urls)} URLs")
+
+        # Filter to URLs you care about
+        blog_urls = [url for url in all_urls if "/blog/" in url]
+        print(f"Found {len(blog_urls)} blog posts to process")
+
+        # ═══════════════════════════════════════════════
+        # Phase 2: Full processing on selected URLs only
+        # ═══════════════════════════════════════════════
+        full_config = CrawlerRunConfig(
+            # Your normal extraction settings
+            word_count_threshold=100,
+            remove_overlay_elements=True,
+        )
+
+        results = []
+        for url in blog_urls:
+            result = await crawler.arun(url, config=full_config)
+            if result.success:
+                results.append(result)
+                print(f"Processed: {url}")
+
+        return results
+
+if __name__ == "__main__":
+    results = asyncio.run(two_phase_crawl("https://example.com"))
+    print(f"Fully processed {len(results)} pages")
+```
+
+### 11.5 Use Cases
+
+- **Site mapping**: Quickly discover all URLs before deciding what to process
+- **Link validation**: Check which pages exist without heavy processing
+- **Selective deep crawl**: Prefetch to find URLs, filter by pattern, then full crawl
+- **Crawl planning**: Estimate crawl size before committing resources
+
+---
+
+## 12. Summary & Next Steps
 
 In this **Deep Crawling with Crawl4AI** tutorial, you learned to:
 
@@ -495,5 +739,7 @@ In this **Deep Crawling with Crawl4AI** tutorial, you learned to:
 - Use scorers to prioritize the most relevant pages
 - Limit crawls with `max_pages` and `score_threshold` parameters
 - Build a complete advanced crawler with combined techniques
+- **Implement crash recovery** with `resume_state` and `on_state_change` for production deployments
+- **Use prefetch mode** for fast URL discovery and two-phase crawling
 
 With these tools, you can efficiently extract structured data from websites at scale, focusing precisely on the content you need for your specific use case.

docs/md_v2/core/examples.md

@@ -11,6 +11,12 @@ This page provides a comprehensive list of example scripts that demonstrate vari
 | Quickstart Set 1 | Basic examples for getting started with Crawl4AI. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/quickstart_examples_set_1.py) |
 | Quickstart Set 2 | More advanced examples for working with Crawl4AI. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/quickstart_examples_set_2.py) |
 
+## Proxies
+
+| Example  | Description | Link |
+|----------|--------------|------|
+| **NSTProxy** | [NSTProxy](https://www.nstproxy.com/?utm_source=crawl4ai) Seamlessly integrates with crawl4ai — no setup required. Access high-performance residential, datacenter, ISP, and IPv6 proxies with smart rotation and anti-blocking technology. Starts from $0.1/GB. Use code crawl4ai for 10% off. | [View Code](https://github.com/unclecode/crawl4ai/tree/main/docs/examples/proxy) |
+
 ## Browser & Crawling Features
 
 | Example | Description | Link |
@@ -56,13 +62,14 @@ This page provides a comprehensive list of example scripts that demonstrate vari
 
 ## Anti-Bot & Stealth Features
 
-| Example | Description | Link |
-|---------|-------------|------|
-| Stealth Mode Quick Start | Five practical examples showing how to use stealth mode for bypassing basic bot detection. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/stealth_mode_quick_start.py) |
+| Example                    | Description | Link |
+|----------------------------|-------------|------|
+| Stealth Mode Quick Start   | Five practical examples showing how to use stealth mode for bypassing basic bot detection. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/stealth_mode_quick_start.py) |
 | Stealth Mode Comprehensive | Comprehensive demonstration of stealth mode features with bot detection testing and comparisons. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/stealth_mode_example.py) |
-| Undetected Browser | Simple example showing how to use the undetected browser adapter. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/hello_world_undetected.py) |
-| Undetected Browser Demo | Basic demo comparing regular and undetected browser modes. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/undetected_simple_demo.py) |
-| Undetected Tests | Advanced tests comparing regular vs undetected browsers on various bot detection services. | [View Folder](https://github.com/unclecode/crawl4ai/tree/main/docs/examples/undetectability/) |
+| Undetected Browser         | Simple example showing how to use the undetected browser adapter. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/hello_world_undetected.py) |
+| Undetected Browser Demo    | Basic demo comparing regular and undetected browser modes. | [View Code](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/undetected_simple_demo.py) |
+| Undetected Tests           | Advanced tests comparing regular vs undetected browsers on various bot detection services. | [View Folder](https://github.com/unclecode/crawl4ai/tree/main/docs/examples/undetectability/) |
+| CapSolver Captcha Solver   | Seamlessly integrate with [CapSolver](https://www.capsolver.com/?utm_source=crawl4ai&utm_medium=github_pr&utm_campaign=crawl4ai_integration) to automatically solve reCAPTCHA v2/v3, Cloudflare Turnstile / Challenges, AWS WAF and more for uninterrupted scraping and automation. | [View Folder](https://github.com/unclecode/crawl4ai/tree/main/docs/examples/capsolver_captcha_solver/) |
 
 ## Customization & Security
 

docs/md_v2/core/self-hosting.md

@@ -22,18 +22,6 @@ When you self-host, you can scale from a single container to a full browser infr
   - [Option 1: Using Pre-built Docker Hub Images (Recommended)](#option-1-using-pre-built-docker-hub-images-recommended)
   - [Option 2: Using Docker Compose](#option-2-using-docker-compose)
   - [Option 3: Manual Local Build & Run](#option-3-manual-local-build--run)
-- [Dockerfile Parameters](#dockerfile-parameters)
-- [Using the API](#using-the-api)
-  - [Playground Interface](#playground-interface)
-  - [Python SDK](#python-sdk)
-  - [Understanding Request Schema](#understanding-request-schema)
-  - [REST API Examples](#rest-api-examples)
-- [Additional API Endpoints](#additional-api-endpoints)
-  - [HTML Extraction Endpoint](#html-extraction-endpoint)
-  - [Screenshot Endpoint](#screenshot-endpoint)
-  - [PDF Export Endpoint](#pdf-export-endpoint)
-  - [JavaScript Execution Endpoint](#javascript-execution-endpoint)
-  - [Library Context Endpoint](#library-context-endpoint)
 - [MCP (Model Context Protocol) Support](#mcp-model-context-protocol-support)
   - [What is MCP?](#what-is-mcp)
   - [Connecting via MCP](#connecting-via-mcp)
@@ -79,13 +67,13 @@ Pull and run images directly from Docker Hub without building locally.
 
 #### 1. Pull the Image
 
-Our latest release is `0.7.3`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system.
+Our latest release is `0.8.0`. Images are built with multi-arch manifests, so Docker automatically pulls the correct version for your system.
 
-> 💡 **Note**: The `latest` tag points to the stable `0.7.3` version.
+> 💡 **Note**: The `latest` tag points to the stable `0.8.0` version.
 
 ```bash
 # Pull the latest version
-docker pull unclecode/crawl4ai:0.7.3
+docker pull unclecode/crawl4ai:0.8.0
 
 # Or pull using the latest tag
 docker pull unclecode/crawl4ai:latest
@@ -157,7 +145,7 @@ docker stop crawl4ai && docker rm crawl4ai
 #### Docker Hub Versioning Explained
 
 *   **Image Name:** `unclecode/crawl4ai`
-*   **Tag Format:** `LIBRARY_VERSION[-SUFFIX]` (e.g., `0.7.3`)
+*   **Tag Format:** `LIBRARY_VERSION[-SUFFIX]` (e.g., `0.8.0`)
     *   `LIBRARY_VERSION`: The semantic version of the core `crawl4ai` Python library
     *   `SUFFIX`: Optional tag for release candidates (``) and revisions (`r1`)
 *   **`latest` Tag:** Points to the most recent stable version
@@ -853,6 +841,733 @@ else:
 
 > 💡 **Remember**: Always test your hooks on safe, known websites first before using them on production sites. Never crawl sites that you don't have permission to access or that might be malicious.
 
+### Hooks Utility: Function-Based Approach (Python)
+
+For Python developers, Crawl4AI provides a more convenient way to work with hooks using the `hooks_to_string()` utility function and Docker client integration.
+
+#### Why Use Function-Based Hooks?
+
+**String-Based Approach (shown above)**:
+```python
+hooks_code = {
+    "on_page_context_created": """
+async def hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+"""
+}
+```
+
+**Function-Based Approach (recommended for Python)**:
+```python
+from crawl4ai import Crawl4aiDockerClient
+
+async def my_hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    return page
+
+async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client:
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks={"on_page_context_created": my_hook}
+    )
+```
+
+**Benefits**:
+- ✅ Write hooks as regular Python functions
+- ✅ Full IDE support (autocomplete, syntax highlighting, type checking)
+- ✅ Easy to test and debug
+- ✅ Reusable hook libraries
+- ✅ Automatic conversion to API format
+
+#### Using the Hooks Utility
+
+The `hooks_to_string()` utility converts Python function objects to the string format required by the API:
+
+```python
+from crawl4ai import hooks_to_string
+
+# Define your hooks as functions
+async def setup_hook(page, context, **kwargs):
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+    await context.add_cookies([{
+        "name": "session",
+        "value": "token",
+        "domain": ".example.com"
+    }])
+    return page
+
+async def scroll_hook(page, context, **kwargs):
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    return page
+
+# Convert to string format
+hooks_dict = {
+    "on_page_context_created": setup_hook,
+    "before_retrieve_html": scroll_hook
+}
+hooks_string = hooks_to_string(hooks_dict)
+
+# Now use with REST API or Docker client
+# hooks_string contains the string representations
+```
+
+#### Docker Client with Automatic Conversion
+
+The Docker client automatically detects and converts function objects:
+
+```python
+from crawl4ai import Crawl4aiDockerClient
+
+async def auth_hook(page, context, **kwargs):
+    """Add authentication cookies"""
+    await context.add_cookies([{
+        "name": "auth_token",
+        "value": "your_token",
+        "domain": ".example.com"
+    }])
+    return page
+
+async def performance_hook(page, context, **kwargs):
+    """Block unnecessary resources"""
+    await context.route("**/*.{png,jpg,gif}", lambda r: r.abort())
+    await context.route("**/analytics/*", lambda r: r.abort())
+    return page
+
+async with Crawl4aiDockerClient(base_url="http://localhost:11235") as client:
+    # Pass functions directly - automatic conversion!
+    result = await client.crawl(
+        ["https://example.com"],
+        hooks={
+            "on_page_context_created": performance_hook,
+            "before_goto": auth_hook
+        },
+        hooks_timeout=30  # Optional timeout in seconds (1-120)
+    )
+
+    print(f"Success: {result.success}")
+    print(f"HTML: {len(result.html)} chars")
+```
+
+#### Creating Reusable Hook Libraries
+
+Build collections of reusable hooks:
+
+```python
+# hooks_library.py
+class CrawlHooks:
+    """Reusable hook collection for common crawling tasks"""
+
+    @staticmethod
+    async def block_images(page, context, **kwargs):
+        """Block all images to speed up crawling"""
+        await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda r: r.abort())
+        return page
+
+    @staticmethod
+    async def block_analytics(page, context, **kwargs):
+        """Block analytics and tracking scripts"""
+        tracking_domains = [
+            "**/google-analytics.com/*",
+            "**/googletagmanager.com/*",
+            "**/facebook.com/tr/*",
+            "**/doubleclick.net/*"
+        ]
+        for domain in tracking_domains:
+            await context.route(domain, lambda r: r.abort())
+        return page
+
+    @staticmethod
+    async def scroll_infinite(page, context, **kwargs):
+        """Handle infinite scroll to load more content"""
+        previous_height = 0
+        for i in range(5):  # Max 5 scrolls
+            current_height = await page.evaluate("document.body.scrollHeight")
+            if current_height == previous_height:
+                break
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await page.wait_for_timeout(1000)
+            previous_height = current_height
+        return page
+
+    @staticmethod
+    async def wait_for_dynamic_content(page, context, url, response, **kwargs):
+        """Wait for dynamic content to load"""
+        await page.wait_for_timeout(2000)
+        try:
+            # Click "Load More" if present
+            load_more = await page.query_selector('[class*="load-more"]')
+            if load_more:
+                await load_more.click()
+                await page.wait_for_timeout(1000)
+        except:
+            pass
+        return page
+
+# Use in your application
+from hooks_library import CrawlHooks
+from crawl4ai import Crawl4aiDockerClient
+
+async def crawl_with_optimizations(url):
+    async with Crawl4aiDockerClient() as client:
+        result = await client.crawl(
+            [url],
+            hooks={
+                "on_page_context_created": CrawlHooks.block_images,
+                "before_retrieve_html": CrawlHooks.scroll_infinite
+            }
+        )
+        return result
+```
+
+#### Choosing the Right Approach
+
+| Approach | Best For | IDE Support | Language |
+|----------|----------|-------------|----------|
+| **String-based** | Non-Python clients, REST APIs, other languages | ❌ None | Any |
+| **Function-based** | Python applications, local development | ✅ Full | Python only |
+| **Docker Client** | Python apps with automatic conversion | ✅ Full | Python only |
+
+**Recommendation**:
+- **Python applications**: Use Docker client with function objects (easiest)
+- **Non-Python or REST API**: Use string-based hooks (most flexible)
+- **Manual control**: Use `hooks_to_string()` utility (middle ground)
+
+#### Complete Example with Function Hooks
+
+```python
+from crawl4ai import Crawl4aiDockerClient, BrowserConfig, CrawlerRunConfig, CacheMode
+
+# Define hooks as regular Python functions
+async def setup_environment(page, context, **kwargs):
+    """Setup crawling environment"""
+    # Set viewport
+    await page.set_viewport_size({"width": 1920, "height": 1080})
+
+    # Block resources for speed
+    await context.route("**/*.{png,jpg,gif}", lambda r: r.abort())
+
+    # Add custom headers
+    await page.set_extra_http_headers({
+        "Accept-Language": "en-US",
+        "X-Custom-Header": "Crawl4AI"
+    })
+
+    print("[HOOK] Environment configured")
+    return page
+
+async def extract_content(page, context, **kwargs):
+    """Extract and prepare content"""
+    # Scroll to load lazy content
+    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+    await page.wait_for_timeout(1000)
+
+    # Extract metadata
+    metadata = await page.evaluate('''() => ({
+        title: document.title,
+        links: document.links.length,
+        images: document.images.length
+    })''')
+
+    print(f"[HOOK] Page metadata: {metadata}")
+    return page
+
+async def main():
+    async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client:
+        # Configure crawl
+        browser_config = BrowserConfig(headless=True)
+        crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
+
+        # Crawl with hooks
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            browser_config=browser_config,
+            crawler_config=crawler_config,
+            hooks={
+                "on_page_context_created": setup_environment,
+                "before_retrieve_html": extract_content
+            },
+            hooks_timeout=30
+        )
+
+        if result.success:
+            print(f"✅ Crawl successful!")
+            print(f"   URL: {result.url}")
+            print(f"   HTML: {len(result.html)} chars")
+            print(f"   Markdown: {len(result.markdown)} chars")
+        else:
+            print(f"❌ Crawl failed: {result.error_message}")
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())
+```
+
+#### Additional Resources
+
+- **Comprehensive Examples**: See `/docs/examples/hooks_docker_client_example.py` for Python function-based examples
+- **REST API Examples**: See `/docs/examples/hooks_rest_api_example.py` for string-based examples
+- **Comparison Guide**: See `/docs/examples/README_HOOKS.md` for detailed comparison
+- **Utility Documentation**: See `/docs/hooks-utility-guide.md` for complete guide
+
+---
+
+## Job Queue & Webhook API
+
+The Docker deployment includes a powerful asynchronous job queue system with webhook support for both crawling and LLM extraction tasks. Instead of waiting for long-running operations to complete, submit jobs and receive real-time notifications via webhooks when they finish.
+
+### Why Use the Job Queue API?
+
+**Traditional Synchronous API (`/crawl`):**
+- Client waits for entire crawl to complete
+- Timeout issues with long-running crawls
+- Resource blocking during execution
+- Constant polling required for status updates
+
+**Asynchronous Job Queue API (`/crawl/job`, `/llm/job`):**
+- ✅ Submit job and continue immediately
+- ✅ No timeout concerns for long operations
+- ✅ Real-time webhook notifications on completion
+- ✅ Better resource utilization
+- ✅ Perfect for batch processing
+- ✅ Ideal for microservice architectures
+
+### Available Endpoints
+
+#### 1. Crawl Job Endpoint
+
+```
+POST /crawl/job
+```
+
+Submit an asynchronous crawl job with optional webhook notification.
+
+**Request Body:**
+```json
+{
+  "urls": ["https://example.com"],
+  "cache_mode": "bypass",
+  "extraction_strategy": {
+    "type": "JsonCssExtractionStrategy",
+    "schema": {
+      "title": "h1",
+      "content": ".article-body"
+    }
+  },
+  "webhook_config": {
+    "webhook_url": "https://your-app.com/webhook/crawl-complete",
+    "webhook_data_in_payload": true,
+    "webhook_headers": {
+      "X-Webhook-Secret": "your-secret-token",
+      "X-Custom-Header": "value"
+    }
+  }
+}
+```
+
+**Response:**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "message": "Crawl job submitted"
+}
+```
+
+#### 2. LLM Extraction Job Endpoint
+
+```
+POST /llm/job
+```
+
+Submit an asynchronous LLM extraction job with optional webhook notification.
+
+**Request Body:**
+```json
+{
+  "url": "https://example.com/article",
+  "q": "Extract the article title, author, publication date, and main points",
+  "provider": "openai/gpt-4o-mini",
+  "schema": "{\"title\": \"string\", \"author\": \"string\", \"date\": \"string\", \"points\": [\"string\"]}",
+  "cache": false,
+  "webhook_config": {
+    "webhook_url": "https://your-app.com/webhook/llm-complete",
+    "webhook_data_in_payload": true,
+    "webhook_headers": {
+      "X-Webhook-Secret": "your-secret-token"
+    }
+  }
+}
+```
+
+**Response:**
+```json
+{
+  "task_id": "llm_1698765432",
+  "message": "LLM job submitted"
+}
+```
+
+#### 3. Job Status Endpoint
+
+```
+GET /job/{task_id}
+```
+
+Check the status and retrieve results of a submitted job.
+
+**Response (In Progress):**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "status": "processing",
+  "message": "Job is being processed"
+}
+```
+
+**Response (Completed):**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "status": "completed",
+  "result": {
+    "markdown": "# Page Title\n\nContent...",
+    "extracted_content": {...},
+    "links": {...}
+  }
+}
+```
+
+### Webhook Configuration
+
+Webhooks provide real-time notifications when your jobs complete, eliminating the need for constant polling.
+
+#### Webhook Config Parameters
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `webhook_url` | string | Yes | Your HTTP(S) endpoint to receive notifications |
+| `webhook_data_in_payload` | boolean | No | Include full result data in webhook payload (default: false) |
+| `webhook_headers` | object | No | Custom headers for authentication/identification |
+
+#### Webhook Payload Format
+
+**Success Notification (Crawl Job):**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "task_type": "crawl",
+  "status": "completed",
+  "timestamp": "2025-10-22T12:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "data": {
+    "markdown": "# Page content...",
+    "extracted_content": {...},
+    "links": {...}
+  }
+}
+```
+
+**Success Notification (LLM Job):**
+```json
+{
+  "task_id": "llm_1698765432",
+  "task_type": "llm_extraction",
+  "status": "completed",
+  "timestamp": "2025-10-22T12:30:00.000000+00:00",
+  "urls": ["https://example.com/article"],
+  "data": {
+    "extracted_content": {
+      "title": "Understanding Web Scraping",
+      "author": "John Doe",
+      "date": "2025-10-22",
+      "points": ["Point 1", "Point 2"]
+    }
+  }
+}
+```
+
+**Failure Notification:**
+```json
+{
+  "task_id": "crawl_1698765432",
+  "task_type": "crawl",
+  "status": "failed",
+  "timestamp": "2025-10-22T12:30:00.000000+00:00",
+  "urls": ["https://example.com"],
+  "error": "Connection timeout after 30 seconds"
+}
+```
+
+#### Webhook Delivery & Retry
+
+- **Delivery Method:** HTTP POST to your `webhook_url`
+- **Content-Type:** `application/json`
+- **Retry Policy:** Exponential backoff with 5 attempts
+  - Attempt 1: Immediate
+  - Attempt 2: 1 second delay
+  - Attempt 3: 2 seconds delay
+  - Attempt 4: 4 seconds delay
+  - Attempt 5: 8 seconds delay
+- **Success Status Codes:** 200-299
+- **Custom Headers:** Your `webhook_headers` are included in every request
+
+### Usage Examples
+
+#### Example 1: Python with Webhook Handler (Flask)
+
+```python
+from flask import Flask, request, jsonify
+import requests
+
+app = Flask(__name__)
+
+# Webhook handler
+@app.route('/webhook/crawl-complete', methods=['POST'])
+def handle_crawl_webhook():
+    payload = request.json
+
+    if payload['status'] == 'completed':
+        print(f"✅ Job {payload['task_id']} completed!")
+        print(f"Task type: {payload['task_type']}")
+
+        # Access the crawl results
+        if 'data' in payload:
+            markdown = payload['data'].get('markdown', '')
+            extracted = payload['data'].get('extracted_content', {})
+            print(f"Extracted {len(markdown)} characters")
+            print(f"Structured data: {extracted}")
+    else:
+        print(f"❌ Job {payload['task_id']} failed: {payload.get('error')}")
+
+    return jsonify({"status": "received"}), 200
+
+# Submit a crawl job with webhook
+def submit_crawl_job():
+    response = requests.post(
+        "http://localhost:11235/crawl/job",
+        json={
+            "urls": ["https://example.com"],
+            "extraction_strategy": {
+                "type": "JsonCssExtractionStrategy",
+                "schema": {
+                    "name": "Example Schema",
+                    "baseSelector": "body",
+                    "fields": [
+                        {"name": "title", "selector": "h1", "type": "text"},
+                        {"name": "description", "selector": "meta[name='description']", "type": "attribute", "attribute": "content"}
+                    ]
+                }
+            },
+            "webhook_config": {
+                "webhook_url": "https://your-app.com/webhook/crawl-complete",
+                "webhook_data_in_payload": True,
+                "webhook_headers": {
+                    "X-Webhook-Secret": "your-secret-token"
+                }
+            }
+        }
+    )
+
+    task_id = response.json()['task_id']
+    print(f"Job submitted: {task_id}")
+    return task_id
+
+if __name__ == '__main__':
+    app.run(port=5000)
+```
+
+#### Example 2: LLM Extraction with Webhooks
+
+```python
+import requests
+
+def submit_llm_job_with_webhook():
+    response = requests.post(
+        "http://localhost:11235/llm/job",
+        json={
+            "url": "https://example.com/article",
+            "q": "Extract the article title, author, and main points",
+            "provider": "openai/gpt-4o-mini",
+            "webhook_config": {
+                "webhook_url": "https://your-app.com/webhook/llm-complete",
+                "webhook_data_in_payload": True,
+                "webhook_headers": {
+                    "X-Webhook-Secret": "your-secret-token"
+                }
+            }
+        }
+    )
+
+    task_id = response.json()['task_id']
+    print(f"LLM job submitted: {task_id}")
+    return task_id
+
+# Webhook handler for LLM jobs
+@app.route('/webhook/llm-complete', methods=['POST'])
+def handle_llm_webhook():
+    payload = request.json
+
+    if payload['status'] == 'completed':
+        extracted = payload['data']['extracted_content']
+        print(f"✅ LLM extraction completed!")
+        print(f"Results: {extracted}")
+    else:
+        print(f"❌ LLM extraction failed: {payload.get('error')}")
+
+    return jsonify({"status": "received"}), 200
+```
+
+#### Example 3: Without Webhooks (Polling)
+
+If you don't use webhooks, you can poll for results:
+
+```python
+import requests
+import time
+
+# Submit job
+response = requests.post(
+    "http://localhost:11235/crawl/job",
+    json={"urls": ["https://example.com"]}
+)
+task_id = response.json()['task_id']
+
+# Poll for results
+while True:
+    result = requests.get(f"http://localhost:11235/job/{task_id}")
+    data = result.json()
+
+    if data['status'] == 'completed':
+        print("Job completed!")
+        print(data['result'])
+        break
+    elif data['status'] == 'failed':
+        print(f"Job failed: {data.get('error')}")
+        break
+
+    print("Still processing...")
+    time.sleep(2)
+```
+
+#### Example 4: Global Webhook Configuration
+
+Set a default webhook URL in your `config.yml` to avoid repeating it in every request:
+
+```yaml
+# config.yml
+api:
+  crawler:
+    # ... other settings ...
+    webhook:
+      default_url: "https://your-app.com/webhook/default"
+      default_headers:
+        X-Webhook-Secret: "your-secret-token"
+```
+
+Then submit jobs without webhook config:
+
+```python
+# Uses the global webhook configuration
+response = requests.post(
+    "http://localhost:11235/crawl/job",
+    json={"urls": ["https://example.com"]}
+)
+```
+
+### Webhook Best Practices
+
+1. **Authentication:** Always use custom headers for webhook authentication
+   ```json
+   "webhook_headers": {
+     "X-Webhook-Secret": "your-secret-token"
+   }
+   ```
+
+2. **Idempotency:** Design your webhook handler to be idempotent (safe to receive duplicate notifications)
+
+3. **Fast Response:** Return HTTP 200 quickly; process data asynchronously if needed
+   ```python
+   @app.route('/webhook', methods=['POST'])
+   def webhook():
+       payload = request.json
+       # Queue for background processing
+       queue.enqueue(process_webhook, payload)
+       return jsonify({"status": "received"}), 200
+   ```
+
+4. **Error Handling:** Handle both success and failure notifications
+   ```python
+   if payload['status'] == 'completed':
+       # Process success
+   elif payload['status'] == 'failed':
+       # Log error, retry, or alert
+   ```
+
+5. **Validation:** Verify webhook authenticity using custom headers
+   ```python
+   secret = request.headers.get('X-Webhook-Secret')
+   if secret != os.environ['EXPECTED_SECRET']:
+       return jsonify({"error": "Unauthorized"}), 401
+   ```
+
+6. **Logging:** Log webhook deliveries for debugging
+   ```python
+   logger.info(f"Webhook received: {payload['task_id']} - {payload['status']}")
+   ```
+
+### Use Cases
+
+**1. Batch Processing**
+Submit hundreds of URLs and get notified as each completes:
+```python
+urls = ["https://site1.com", "https://site2.com", ...]
+for url in urls:
+    submit_crawl_job(url, webhook_url="https://app.com/webhook")
+```
+
+**2. Microservice Integration**
+Integrate with event-driven architectures:
+```python
+# Service A submits job
+task_id = submit_crawl_job(url)
+
+# Service B receives webhook and triggers next step
+@app.route('/webhook')
+def webhook():
+    process_result(request.json)
+    trigger_next_service()
+    return "OK", 200
+```
+
+**3. Long-Running Extractions**
+Handle complex LLM extractions without timeouts:
+```python
+submit_llm_job(
+    url="https://long-article.com",
+    q="Comprehensive summary with key points and analysis",
+    webhook_url="https://app.com/webhook/llm"
+)
+```
+
+### Troubleshooting
+
+**Webhook not receiving notifications?**
+- Check your webhook URL is publicly accessible
+- Verify firewall/security group settings
+- Use webhook testing tools like webhook.site for debugging
+- Check server logs for delivery attempts
+- Ensure your handler returns 200-299 status code
+
+**Job stuck in processing?**
+- Check Redis connection: `docker logs <container_name> | grep redis`
+- Verify worker processes: `docker exec <container_name> ps aux | grep worker`
+- Check server logs: `docker logs <container_name>`
+
+**Need to cancel a job?**
+Jobs are processed asynchronously. If you need to cancel:
+- Delete the task from Redis (requires Redis CLI access)
+- Or implement a cancellation endpoint in your webhook handler
+
 ---
 
 ## Dockerfile Parameters
@@ -913,10 +1628,12 @@ This is the easiest way to translate Python configuration to JSON requests when
 
 Install the SDK: `pip install crawl4ai`
 
+The Python SDK provides a convenient way to interact with the Docker API, including **automatic hook conversion** when using function objects.
+
 ```python
 import asyncio
 from crawl4ai.docker_client import Crawl4aiDockerClient
-from crawl4ai import BrowserConfig, CrawlerRunConfig, CacheMode # Assuming you have crawl4ai installed
+from crawl4ai import BrowserConfig, CrawlerRunConfig, CacheMode
 
 async def main():
     # Point to the correct server port
@@ -928,23 +1645,22 @@ async def main():
         print("--- Running Non-Streaming Crawl ---")
         results = await client.crawl(
             ["https://httpbin.org/html"],
-            browser_config=BrowserConfig(headless=True), # Use library classes for config aid
+            browser_config=BrowserConfig(headless=True),
             crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
         )
-        if results: # client.crawl returns None on failure
-          print(f"Non-streaming results success: {results.success}")
-          if results.success:
-              for result in results: # Iterate through the CrawlResultContainer
-                  print(f"URL: {result.url}, Success: {result.success}")
+        if results:
+            print(f"Non-streaming results success: {results.success}")
+            if results.success:
+                for result in results:
+                    print(f"URL: {result.url}, Success: {result.success}")
         else:
             print("Non-streaming crawl failed.")
 
-
         # Example Streaming crawl
         print("\n--- Running Streaming Crawl ---")
         stream_config = CrawlerRunConfig(stream=True, cache_mode=CacheMode.BYPASS)
         try:
-            async for result in await client.crawl( # client.crawl returns an async generator for streaming
+            async for result in await client.crawl(
                 ["https://httpbin.org/html", "https://httpbin.org/links/5/0"],
                 browser_config=BrowserConfig(headless=True),
                 crawler_config=stream_config
@@ -953,17 +1669,56 @@ async def main():
         except Exception as e:
             print(f"Streaming crawl failed: {e}")
 
+        # Example with hooks (Python function objects)
+        print("\n--- Crawl with Hooks ---")
+
+        async def my_hook(page, context, **kwargs):
+            """Custom hook to optimize performance"""
+            await page.set_viewport_size({"width": 1920, "height": 1080})
+            await context.route("**/*.{png,jpg}", lambda r: r.abort())
+            print("[HOOK] Page optimized")
+            return page
+
+        result = await client.crawl(
+            ["https://httpbin.org/html"],
+            browser_config=BrowserConfig(headless=True),
+            crawler_config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS),
+            hooks={"on_page_context_created": my_hook},  # Pass function directly!
+            hooks_timeout=30
+        )
+        print(f"Crawl with hooks success: {result.success}")
 
         # Example Get schema
         print("\n--- Getting Schema ---")
         schema = await client.get_schema()
-        print(f"Schema received: {bool(schema)}") # Print whether schema was received
+        print(f"Schema received: {bool(schema)}")
 
 if __name__ == "__main__":
     asyncio.run(main())
 ```
 
-*(SDK parameters like timeout, verify_ssl etc. remain the same)*
+#### SDK Parameters
+
+The Docker client supports the following parameters:
+
+**Client Initialization**:
+- `base_url` (str): URL of the Docker server (default: `http://localhost:8000`)
+- `timeout` (float): Request timeout in seconds (default: 30.0)
+- `verify_ssl` (bool): Verify SSL certificates (default: True)
+- `verbose` (bool): Enable verbose logging (default: True)
+- `log_file` (Optional[str]): Path to log file (default: None)
+
+**crawl() Method**:
+- `urls` (List[str]): List of URLs to crawl
+- `browser_config` (Optional[BrowserConfig]): Browser configuration
+- `crawler_config` (Optional[CrawlerRunConfig]): Crawler configuration
+- `hooks` (Optional[Dict]): Hook functions or strings - **automatically converts function objects!**
+- `hooks_timeout` (int): Timeout for each hook execution in seconds (default: 30)
+
+**Returns**:
+- Single URL: `CrawlResult` object
+- Multiple URLs: `List[CrawlResult]`
+- Streaming: `AsyncGenerator[CrawlResult]`
 
 ### Second Approach: Direct API Calls
 

docs/md_v2/core/url-seeding.md

@@ -255,6 +255,8 @@ The `SeedingConfig` object is your control panel. Here's everything you can conf
 | `scoring_method` | str | None | Scoring method (currently "bm25") |
 | `score_threshold` | float | None | Minimum score to include URL |
 | `filter_nonsense_urls` | bool | True | Filter out utility URLs (robots.txt, etc.) |
+| `cache_ttl_hours` | int | 24 | Hours before sitemap cache expires (0 = no TTL) |
+| `validate_sitemap_lastmod` | bool | True | Check sitemap's lastmod and refetch if newer |
 
 #### Pattern Matching Examples
 
@@ -968,10 +970,49 @@ config = SeedingConfig(
 The seeder automatically caches results to speed up repeated operations:
 
 - **Common Crawl cache**: `~/.crawl4ai/seeder_cache/[index]_[domain]_[hash].jsonl`
-- **Sitemap cache**: `~/.crawl4ai/seeder_cache/sitemap_[domain]_[hash].jsonl`
+- **Sitemap cache**: `~/.crawl4ai/seeder_cache/sitemap_[domain]_[hash].json`
 - **HEAD data cache**: `~/.cache/url_seeder/head/[hash].json`
 
-Cache expires after 7 days by default. Use `force=True` to refresh.
+#### Smart TTL Cache for Sitemaps
+
+Sitemap caches now include intelligent validation:
+
+```python
+# Default: 24-hour TTL with lastmod validation
+config = SeedingConfig(
+    source="sitemap",
+    cache_ttl_hours=24,              # Cache expires after 24 hours
+    validate_sitemap_lastmod=True    # Also check if sitemap was updated
+)
+
+# Aggressive caching (1 week, no lastmod check)
+config = SeedingConfig(
+    source="sitemap",
+    cache_ttl_hours=168,             # 7 days
+    validate_sitemap_lastmod=False   # Trust TTL only
+)
+
+# Always validate (no TTL, only lastmod)
+config = SeedingConfig(
+    source="sitemap",
+    cache_ttl_hours=0,               # Disable TTL
+    validate_sitemap_lastmod=True    # Refetch if sitemap has newer lastmod
+)
+
+# Always fresh (bypass cache completely)
+config = SeedingConfig(
+    source="sitemap",
+    force=True                       # Ignore all caching
+)
+```
+
+**Cache validation priority:**
+1. `force=True` → Always refetch
+2. Cache doesn't exist → Fetch fresh
+3. `validate_sitemap_lastmod=True` and sitemap has newer `<lastmod>` → Refetch
+4. `cache_ttl_hours > 0` and cache is older than TTL → Refetch
+5. Cache corrupted → Refetch (automatic recovery)
+6. Otherwise → Use cache
 
 ### Pattern Matching Strategies
 
@@ -1060,6 +1101,9 @@ config = SeedingConfig(
 | Rate limit errors | Reduce `hits_per_sec` and `concurrency` |
 | Memory issues with large sites | Use `max_urls` to limit results, reduce `concurrency` |
 | Connection not closed | Use context manager or call `await seeder.close()` |
+| Stale/outdated URLs | Set `cache_ttl_hours=0` or use `force=True` |
+| Cache not updating | Check `validate_sitemap_lastmod=True`, or use `force=True` |
+| Incomplete URL list | Delete cache file and refetch, or use `force=True` |
 
 ### Performance Benchmarks
 
@@ -1119,6 +1163,7 @@ config = SeedingConfig(
 3. **Context Manager Support**: Automatic cleanup with `async with` statement
 4. **URL-Based Scoring**: Smart filtering even without head extraction
 5. **Smart URL Filtering**: Automatically excludes utility/nonsense URLs
-6. **Dual Caching**: Separate caches for URL lists and metadata
+6. **Smart TTL Cache**: Sitemap caches with TTL expiry and lastmod validation
+7. **Automatic Cache Recovery**: Corrupted or incomplete caches are automatically refreshed
 
-Now go forth and seed intelligently! 🌱🚀
+Now go forth and seed intelligently!

docs/md_v2/extraction/llm-strategies.md

@@ -20,10 +20,10 @@ In some cases, you need to extract **complex or unstructured** information from
 
 ## 2. Provider-Agnostic via LiteLLM
 
-You can use LlmConfig, to quickly configure multiple variations of LLMs and experiment with them to find the optimal one for your use case. You can read more about LlmConfig [here](/api/parameters).
+You can use LLMConfig, to quickly configure multiple variations of LLMs and experiment with them to find the optimal one for your use case. You can read more about LLMConfig [here](/api/parameters).
 
 ```python
-llmConfig = LlmConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
+llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv("OPENAI_API_KEY"))
 ```
 
 Crawl4AI uses a “provider string” (e.g., `"openai/gpt-4o"`, `"ollama/llama2.0"`, `"aws/titan"`) to identify your LLM. **Any** model that LiteLLM supports is fair game. You just provide:
@@ -58,7 +58,7 @@ For structured data, `"schema"` is recommended. You provide `schema=YourPydantic
 
 Below is an overview of important LLM extraction parameters. All are typically set inside `LLMExtractionStrategy(...)`. You then put that strategy in your `CrawlerRunConfig(..., extraction_strategy=...)`.
 
-1. **`llmConfig`** (LlmConfig): e.g., `"openai/gpt-4"`, `"ollama/llama2"`.    
+1. **`llm_config`** (LLMConfig): e.g., `"openai/gpt-4"`, `"ollama/llama2"`.
 2. **`schema`** (dict): A JSON schema describing the fields you want. Usually generated by `YourModel.model_json_schema()`.  
 3. **`extraction_type`** (str): `"schema"` or `"block"`.  
 4. **`instruction`** (str): Prompt text telling the LLM what you want extracted. E.g., “Extract these fields as a JSON array.”  
@@ -112,7 +112,7 @@ async def main():
     # 1. Define the LLM extraction strategy
     llm_strategy = LLMExtractionStrategy(
         llm_config = LLMConfig(provider="openai/gpt-4o-mini", api_token=os.getenv('OPENAI_API_KEY')),
-        schema=Product.schema_json(), # Or use model_json_schema()
+        schema=Product.model_json_schema(), # Or use model_json_schema()
         extraction_type="schema",
         instruction="Extract all product objects with 'name' and 'price' from the content.",
         chunk_token_threshold=1000,
@@ -238,7 +238,7 @@ class KnowledgeGraph(BaseModel):
 async def main():
     # LLM extraction strategy
     llm_strat = LLMExtractionStrategy(
-        llmConfig = LLMConfig(provider="openai/gpt-4", api_token=os.getenv('OPENAI_API_KEY')),
+        llm_config = LLMConfig(provider="openai/gpt-4", api_token=os.getenv('OPENAI_API_KEY')),
         schema=KnowledgeGraph.model_json_schema(),
         extraction_type="schema",
         instruction="Extract entities and relationships from the content. Return valid JSON.",

docs/md_v2/extraction/no-llm-strategies.md

@@ -712,10 +712,106 @@ strategy = JsonCssExtractionStrategy(css_schema)
 3. **Consider Both CSS and XPath**: Try both schema types and choose the one that works best for your specific case.
 4. **Cache Generated Schemas**: Since generation uses LLM, save successful schemas for reuse.
 5. **API Token Security**: Never hardcode API tokens. Use environment variables or secure configuration management.
-6. **Choose Provider Wisely**: 
+6. **Choose Provider Wisely**:
    - Use OpenAI for production-quality schemas
    - Use Ollama for development, testing, or when you need a self-hosted solution
 
+### Multi-Sample Schema Generation
+
+When scraping multiple pages with varying DOM structures (e.g., product pages where table rows appear in different positions), single-sample schema generation may produce **fragile selectors** like `tr:nth-child(6)` that break on other pages.
+
+**The Problem:**
+```
+Page A: Manufacturer is in row 6  → selector: tr:nth-child(6) td a
+Page B: Manufacturer is in row 5  → selector FAILS
+Page C: Manufacturer is in row 7  → selector FAILS
+```
+
+**The Solution:** Provide multiple HTML samples so the LLM identifies stable patterns that work across all pages.
+
+```python
+from crawl4ai import JsonCssExtractionStrategy, LLMConfig
+
+# Collect HTML samples from different pages
+html_sample_1 = """
+<table class="specs">
+  <tr><td>Brand</td><td>Apple</td></tr>
+  <tr><td>Manufacturer</td><td><a href="/m/apple">Apple Inc</a></td></tr>
+</table>
+"""
+
+html_sample_2 = """
+<table class="specs">
+  <tr><td>Manufacturer</td><td><a href="/m/samsung">Samsung</a></td></tr>
+  <tr><td>Brand</td><td>Galaxy</td></tr>
+</table>
+"""
+
+html_sample_3 = """
+<table class="specs">
+  <tr><td>Model</td><td>Pixel 8</td></tr>
+  <tr><td>Brand</td><td>Google</td></tr>
+  <tr><td>Manufacturer</td><td><a href="/m/google">Google LLC</a></td></tr>
+</table>
+"""
+
+# Combine samples with labels
+combined_html = """
+## HTML Sample 1 (Product A):
+```html
+""" + html_sample_1 + """
+```
+
+## HTML Sample 2 (Product B):
+```html
+""" + html_sample_2 + """
+```
+
+## HTML Sample 3 (Product C):
+```html
+""" + html_sample_3 + """
+```
+"""
+
+# Provide instructions for stable selectors
+query = """
+IMPORTANT: I'm providing 3 HTML samples from different product pages.
+The manufacturer field appears in different row positions across pages.
+Generate selectors using stable attributes like href patterns (e.g., a[href*='/m/'])
+instead of fragile positional selectors like nth-child().
+Extract: manufacturer name and link.
+"""
+
+# Generate schema with multi-sample awareness
+schema = JsonCssExtractionStrategy.generate_schema(
+    html=combined_html,
+    query=query,
+    schema_type="CSS",
+    llm_config=LLMConfig(provider="openai/gpt-4o", api_token="your-token")
+)
+
+# The generated schema will use stable selectors like:
+# a[href*="/m/"] instead of tr:nth-child(6) td a
+print(schema)
+```
+
+**Key Points for Multi-Sample Queries:**
+
+1. **Format samples clearly** - Use markdown headers and code blocks to separate samples
+2. **State the number of samples** - "I'm providing 3 HTML samples..."
+3. **Explain the variation** - "...the manufacturer field appears in different row positions"
+4. **Request stable selectors** - "Use href patterns, data attributes, or class names instead of nth-child"
+
+**Stable vs Fragile Selectors:**
+
+| Fragile (single sample) | Stable (multi-sample) |
+|------------------------|----------------------|
+| `tr:nth-child(6) td a` | `a[href*="/m/"]` |
+| `div:nth-child(3) .price` | `.price, [data-price]` |
+| `ul li:first-child` | `li[data-featured="true"]` |
+
+This approach lets you generate schemas once that work reliably across hundreds of similar pages with varying structures.
+
 ---
 
 ## 10. Conclusion

docs/md_v2/index.md

@@ -55,9 +55,40 @@
   
 </div>
 
+---
+#### 🚀 Crawl4AI Cloud API — Closed Beta (Launching Soon)
+Reliable, large-scale web extraction, now built to be _**drastically more cost-effective**_ than any of the existing solutions.
+
+👉 **Apply [here](https://forms.gle/E9MyPaNXACnAMaqG7) for early access**  
+_We’ll be onboarding in phases and working closely with early users.
+Limited slots._
+
+---
+
 Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for large language models, AI agents, and data pipelines. Fully open source, flexible, and built for real-time performance, **Crawl4AI** empowers developers with unmatched speed, precision, and deployment ease.
 
-> **Note**: If you're looking for the old documentation, you can access it [here](https://old.docs.crawl4ai.com).
+> Enjoy using Crawl4AI? Consider **[becoming a sponsor](https://github.com/sponsors/unclecode)** to support ongoing development and community growth!
+
+## 🆕 AI Assistant Skill Now Available!
+
+<div style="background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); padding: 20px; border-radius: 10px; margin: 20px 0; box-shadow: 0 4px 6px rgba(0,0,0,0.1);">
+  <h3 style="color: white; margin: 0 0 10px 0;">🤖 Crawl4AI Skill for Claude & AI Assistants</h3>
+  <p style="color: white; margin: 10px 0;">Supercharge your AI coding assistant with complete Crawl4AI knowledge! Download our comprehensive skill package that includes:</p>
+  <ul style="color: white; margin: 10px 0;">
+    <li>📚 Complete SDK reference (23K+ words)</li>
+    <li>🚀 Ready-to-use extraction scripts</li>
+    <li>⚡ Schema generation for efficient scraping</li>
+    <li>🔧 Version 0.7.4 compatible</li>
+  </ul>
+  <div style="text-align: center; margin-top: 15px;">
+    <a href="assets/crawl4ai-skill.zip" download style="background: white; color: #667eea; padding: 12px 30px; border-radius: 5px; text-decoration: none; font-weight: bold; display: inline-block; transition: transform 0.2s;">
+      📦 Download Skill Package
+    </a>
+  </div>
+  <p style="color: white; margin: 15px 0 0 0; font-size: 0.9em; text-align: center;">
+    Works with Claude, Cursor, Windsurf, and other AI coding assistants. Import the .zip file into your AI assistant's skill/knowledge system.
+  </p>
+</div>
 
 ## 🎯 New: Adaptive Web Crawling
 

docs/md_v2/marketplace/admin/admin.js

@@ -529,8 +529,19 @@ class AdminDashboard {
                     </label>
                 </div>
                 <div class="form-group full-width">
-                    <label>Integration Guide</label>
-                    <textarea id="form-integration" rows="10">${app?.integration_guide || ''}</textarea>
+                    <label>Long Description (Markdown - Overview tab)</label>
+                    <textarea id="form-long-description" rows="10" placeholder="Enter detailed description with markdown formatting...">${app?.long_description || ''}</textarea>
+                    <small>Markdown support: **bold**, *italic*, [links](url), # headers, code blocks, lists</small>
+                </div>
+                <div class="form-group full-width">
+                    <label>Integration Guide (Markdown - Integration tab)</label>
+                    <textarea id="form-integration" rows="20" placeholder="Enter integration guide with installation, examples, and code snippets using markdown...">${app?.integration_guide || ''}</textarea>
+                    <small>Single markdown field with installation, examples, and complete guide. Code blocks get auto copy buttons.</small>
+                </div>
+                <div class="form-group full-width">
+                    <label>Documentation (Markdown - Documentation tab)</label>
+                    <textarea id="form-documentation" rows="20" placeholder="Enter documentation with API reference, examples, and best practices using markdown...">${app?.documentation || ''}</textarea>
+                    <small>Full documentation with API reference, examples, best practices, etc.</small>
                 </div>
             </div>
         `;
@@ -712,7 +723,9 @@ class AdminDashboard {
             data.contact_email = document.getElementById('form-email').value;
             data.featured = document.getElementById('form-featured').checked ? 1 : 0;
             data.sponsored = document.getElementById('form-sponsored').checked ? 1 : 0;
+            data.long_description = document.getElementById('form-long-description').value;
             data.integration_guide = document.getElementById('form-integration').value;
+            data.documentation = document.getElementById('form-documentation').value;
         } else if (type === 'articles') {
             data.title = document.getElementById('form-title').value;
             data.slug = this.generateSlug(data.title);